From fda48e002f6cfecb9d511b62045be7ab0f3c5724 Mon Sep 17 00:00:00 2001
From: "Mads M. Pedersen" <mmpe@dtu.dk>
Date: Thu, 2 Nov 2017 09:20:47 +0100
Subject: [PATCH 1/3] improved caching
---
wetb/hawc2/pc_file.py | 56 ++++++++++++------------
wetb/utils/caching.py | 62 ++++++++++++++++++++++-----
wetb/utils/tests/test_caching.py | 38 ++++++++++++++--
wetb/utils/tests/test_files/test2.csv | 5 +++
4 files changed, 120 insertions(+), 41 deletions(-)
create mode 100644 wetb/utils/tests/test_files/test2.csv
diff --git a/wetb/hawc2/pc_file.py b/wetb/hawc2/pc_file.py
index 84b15cc0..fbfe9c56 100644
--- a/wetb/hawc2/pc_file.py
+++ b/wetb/hawc2/pc_file.py
@@ -21,20 +21,12 @@ class PCFile(object):
examples
--------
- >>> pcfile = PCFile("tests/test_files/NREL_5MW_pc.txt", "tests/test_files/NREL_5MW_ae.txt")
- # Same attributes as AEFile
- >>> pcfile.thickness(36) # Interpolated thickness at radius 36
- 23.78048780487805
- >>> pcfile.chord(36) # Interpolated chord at radius 36
- 3.673
- >>> pcfile.pc_set_nr(36) # pc set number at radius 36
- 1
- # Additional attributes
- >>> pcfile.CL(36,10) # CL at radius=36m and AOA=10deg
+ >>> pcfile = PCFile("tests/test_files/NREL_5MW_pc.txt")
+ >>> pcfile.CL(21,10) # CL for thickness 21% and AOA=10deg
1.358
- >>> pcfile.CD(36,10) # CD at radius=36m and AOA=10deg
+ >>> pcfile.CD(21,10) # CD for thickness 21% and AOA=10deg
0.0255
- >>> pcfile.CM(36,10) # CM at radius=36m and AOA=10deg
+ >>> pcfile.CM(21,10) # CM for thickness 21% and AOA=10deg
-0.1103
"""
def __init__(self, filename):
@@ -59,7 +51,7 @@ class PCFile(object):
lptr += n_rows
self.pc_sets[nset] = (np.array(thicknesses), profiles)
- def _Cxxx(self, thickness, alpha, column, ae_set_nr=1, pc_set_nr=1):
+ def _Cxxx(self, thickness, alpha, column, pc_set_nr=1):
thicknesses, profiles = self.pc_sets[pc_set_nr]
index = np.searchsorted(thicknesses, thickness)
if index == 0:
@@ -71,7 +63,7 @@ class PCFile(object):
th0, th1 = thicknesses[index - 1:index + 1]
return Cx0 + (Cx1 - Cx0) * (thickness - th0) / (th1 - th0)
- def _CxxxH2(self, thickness, alpha, column, ae_set_nr=1,pc_set_nr=1):
+ def _CxxxH2(self, thickness, alpha, column, pc_set_nr=1):
thicknesses, profiles = self.pc_sets[pc_set_nr]
index = np.searchsorted(thicknesses, thickness)
if index == 0:
@@ -89,7 +81,7 @@ class PCFile(object):
- def CL(self, thickness, alpha, ae_set_nr=1,pc_set_nr=1):
+ def CL(self, thickness, alpha, pc_set_nr=1):
"""Lift coefficient
Parameters
@@ -98,20 +90,20 @@ class PCFile(object):
thickness [5]
alpha : float
Angle of attack [deg]
- ae_set_nr : int optional
- Aerdynamic set number, default is 1
+ pc_set_nr : int optional
+ pc set number, default is 1, normally obtained from ae-file
Returns
-------
Lift coefficient : float
"""
- return self._Cxxx(thickness, alpha, 1, ae_set_nr, pc_set_nr)
+ return self._Cxxx(thickness, alpha, 1, pc_set_nr)
- def CL_H2(self, thickness, alpha, ae_set_nr=1, pc_set_nr=1):
- return self._CxxxH2(thickness, alpha, 1, ae_set_nr, pc_set_nr)
+ def CL_H2(self, thickness, alpha, pc_set_nr=1):
+ return self._CxxxH2(thickness, alpha, 1, pc_set_nr)
- def CD(self, thickness, alpha, ae_set_nr=1, pc_set_nr=1):
+ def CD(self, thickness, alpha, pc_set_nr=1):
"""Drag coefficient
Parameters
@@ -120,18 +112,26 @@ class PCFile(object):
radius [m]
alpha : float
Angle of attack [deg]
- ae_set_nr : int optional
- Aerdynamic set number, default is 1
+ pc_set_nr : int optional
+ pc set number, default is 1, normally obtained from ae-file
Returns
-------
Drag coefficient : float
"""
- return self._Cxxx(thickness, alpha, 2, ae_set_nr, pc_set_nr)
+ return self._Cxxx(thickness, alpha, 2, pc_set_nr)
- def CM(self, thickness, alpha, ae_set_nr=1, pc_set_nr=1):
- return self._Cxxx(thickness, alpha, 3, ae_set_nr,pc_set_nr)
+ def CM(self, thickness, alpha, pc_set_nr=1):
+ return self._Cxxx(thickness, alpha, 3, pc_set_nr)
if __name__ == "__main__":
- pc = PCFile(r"C:\mmpe\Projects\inflow\Hawc2aero_setup/data/Hawc_pc.b52", r"C:\mmpe\Projects\inflow\Hawc2aero_setup/data/S36_ae_h2.001")
- print (pc)
+ pcfile = PCFile("tests/test_files/NREL_5MW_pc.txt")
+ aefile = AEFile("tests/test_files/NREL_5MW_ae.txt")
+ print (aefile.thickness(36))
+
+ print (pcfile.CL(21,10)) # CL for thickness 21% and AOA=10deg
+ #1.358
+ print (pcfile.CD(21,10)) # CD for thickness 21% and AOA=10deg
+ #0.0255
+ print (pcfile.CM(21,10)) # CM for thickness 21% and AOA=10deg
+ #-0.1103
diff --git a/wetb/utils/caching.py b/wetb/utils/caching.py
index d8cd22e1..008f184f 100644
--- a/wetb/utils/caching.py
+++ b/wetb/utils/caching.py
@@ -96,26 +96,68 @@ def cache_function(f):
class cache_method():
def __init__(self, N):
self.N = N
- self.cache_dict = OrderedDict()
+
def __call__(self, f):
- def wrapped(*args):
+ def wrapped(caller_obj, *args):
name = "_" + f.__name__
arg_id = ";".join([str(a) for a in args])
- if arg_id not in self.cache_dict:
- self.cache_dict[arg_id] = f(*args)
- if len(self.cache_dict)>self.N:
- self.cache_dict.popitem(last=False)
- return self.cache_dict[arg_id]
+ if not hasattr(caller_obj,'%s_cache_dict'%name):
+ setattr(caller_obj,'%s_cache_dict'%name, OrderedDict())
+ cache_dict = getattr(caller_obj,'%s_cache_dict'%name)
+ if arg_id not in cache_dict:
+ cache_dict[arg_id] = f(caller_obj, *args)
+ if len(cache_dict)>self.N:
+ cache_dict.popitem(last=False)
+ return cache_dict[arg_id]
return wrapped
-def cache_binary(f):
+def cache_npsave(f):
def wrap(filename,*args,**kwargs):
np_filename = os.path.splitext(filename)[0] + ".npy"
+ def loadsave():
+ res = f(filename,*args,**kwargs)
+ np.save(np_filename,res)
+ return res
if os.path.isfile(np_filename) and (not os.path.isfile(filename) or os.path.getmtime(np_filename) > os.path.getmtime(filename)):
- return np.load(np_filename)
+ try:
+ return np.load(np_filename)
+ except:
+ return loadsave()
else:
+ return loadsave()
+ return wrap
+
+def cache_npsavez(f):
+ def wrap(filename,*args,**kwargs):
+ np_filename = os.path.splitext(filename)[0] + ".npy.npz"
+ def loadsave():
res = f(filename,*args,**kwargs)
- np.save(np_filename,res)
+ np.savez(np_filename,*res)
return res
+ if os.path.isfile(np_filename) and (not os.path.isfile(filename) or os.path.getmtime(np_filename) > os.path.getmtime(filename)):
+ try:
+ npzfile = np.load(np_filename)
+ return [npzfile['arr_%d'%i] for i in range(len(f.files()))]
+ except:
+ return loadsave()
+ else:
+ return loadsave()
+ return wrap
+
+
+def cache_npsavez_compressed(f):
+ def wrap(filename,*args,**kwargs):
+ np_filename = os.path.splitext(filename)[0] + ".npy.npz"
+ def loadsave():
+ res = f(filename,*args,**kwargs)
+ np.savez_compressed(np_filename,*res)
+ return res
+ if os.path.isfile(np_filename) and (not os.path.isfile(filename) or os.path.getmtime(np_filename) > os.path.getmtime(filename)):
+ try:
+ return [f['arr_%d'%i] for i in range(len(f.files()))]
+ except:
+ return loadsave()
+ else:
+ return loadsave()
return wrap
\ No newline at end of file
diff --git a/wetb/utils/tests/test_caching.py b/wetb/utils/tests/test_caching.py
index 23655a0d..bf308e76 100644
--- a/wetb/utils/tests/test_caching.py
+++ b/wetb/utils/tests/test_caching.py
@@ -18,7 +18,7 @@ import numpy as np
from wetb.utils.timing import get_time
from wetb.utils.caching import cache_function, set_cache_property, cache_method,\
- cache_binary
+ cache_npsavez, cache_npsave, cache_npsavez_compressed
tfp = os.path.dirname(__file__) + "/test_files/"
class Example(object):
@@ -56,10 +56,18 @@ class Example(object):
time.sleep(1)
return x*2
-@cache_binary
+@cache_npsave
def open_csv(filename):
return np.loadtxt(filename)
+@cache_npsavez
+def open_csv2(filename):
+ return np.loadtxt(filename), np.loadtxt(filename)
+
+@cache_npsavez_compressed
+def open_csv3(filename):
+ return np.loadtxt(filename), np.loadtxt(filename)
+
def f(x):
return x ** 2
@@ -115,7 +123,7 @@ class TestCacheProperty(unittest.TestCase):
e.test_cache_property
self.assertAlmostEqual(time.time()-t, 0, places=1)
- def test_cache_binary(self):
+ def test_cache_save(self):
if os.path.isfile(tfp+"test.npy"):
os.remove(tfp+'test.npy')
A = open_csv(tfp + "test.csv")
@@ -126,7 +134,31 @@ class TestCacheProperty(unittest.TestCase):
B = open_csv(tfp + "test.csv")
np.testing.assert_array_equal(A,B)
os.remove(tfp+'test.npy')
+
+ def test_cache_savez(self):
+ if os.path.isfile(tfp+"test.npy.npy"):
+ os.remove(tfp+'test.npy.npy')
+ A = open_csv2(tfp + "test.csv")
+ self.assertTrue(os.path.isfile(tfp+"test.npy.npz"))
+ np.testing.assert_array_equal(A[0],np.loadtxt(tfp + "test.csv"))
+ A[0][0]=-1
+ np.save(tfp+"test.npy",A)
+ B = open_csv(tfp + "test.csv")
+ np.testing.assert_array_equal(A,B)
+ os.remove(tfp+'test.npy')
+ def test_cache_savez_compressed(self):
+ if os.path.isfile(tfp+"test2.npy.npy"):
+ os.remove(tfp+'test2.npy.npy')
+ A = open_csv2(tfp + "test2.csv")
+ self.assertTrue(os.path.isfile(tfp+"test2.npy.npz"))
+ np.testing.assert_array_equal(A[0],np.loadtxt(tfp + "test2.csv"))
+ A[0][0]=-1
+ np.save(tfp+"test2.npy",A)
+ B = open_csv(tfp + "test2.csv")
+ np.testing.assert_array_equal(A,B)
+ os.remove(tfp+'test2.npy')
+
if __name__ == "__main__":
#import sys;sys.argv = ['', 'Test.testName']
unittest.main()
diff --git a/wetb/utils/tests/test_files/test2.csv b/wetb/utils/tests/test_files/test2.csv
new file mode 100644
index 00000000..158e1a69
--- /dev/null
+++ b/wetb/utils/tests/test_files/test2.csv
@@ -0,0 +1,5 @@
+0.000000000000000000e+00 5.000000000000000000e+00
+1.000000000000000000e+00 6.000000000000000000e+00
+2.000000000000000000e+00 7.000000000000000000e+00
+3.000000000000000000e+00 8.000000000000000000e+00
+4.000000000000000000e+00 9.000000000000000000e+00
--
GitLab
From 07595e1eb278795ca2d7ae70b200be9487cf41e8 Mon Sep 17 00:00:00 2001
From: "Mads M. Pedersen" <mmpe@dtu.dk>
Date: Thu, 2 Nov 2017 09:38:49 +0100
Subject: [PATCH 2/3] fixed caching issue
---
wetb/utils/caching.py | 29 +++++++++--------------
wetb/utils/tests/test_caching.py | 34 +++++++++++++++------------
wetb/utils/tests/test_files/test2.csv | 5 ----
3 files changed, 30 insertions(+), 38 deletions(-)
delete mode 100644 wetb/utils/tests/test_files/test2.csv
diff --git a/wetb/utils/caching.py b/wetb/utils/caching.py
index 008f184f..df0c369f 100644
--- a/wetb/utils/caching.py
+++ b/wetb/utils/caching.py
@@ -128,36 +128,29 @@ def cache_npsave(f):
return loadsave()
return wrap
-def cache_npsavez(f):
+def _get_npsavez_wrap(f, compress):
def wrap(filename,*args,**kwargs):
- np_filename = os.path.splitext(filename)[0] + ".npy.npz"
+ np_filename = os.path.splitext(filename)[0] + ".npy.npz"+("","c")[compress]
def loadsave():
res = f(filename,*args,**kwargs)
- np.savez(np_filename,*res)
+ if compress:
+ np.savez_compressed(np_filename,*res)
+ else:
+ np.savez(np_filename,*res)
return res
if os.path.isfile(np_filename) and (not os.path.isfile(filename) or os.path.getmtime(np_filename) > os.path.getmtime(filename)):
try:
npzfile = np.load(np_filename)
- return [npzfile['arr_%d'%i] for i in range(len(f.files()))]
+ return [npzfile['arr_%d'%i] for i in range(len(npzfile.files))]
except:
return loadsave()
else:
return loadsave()
return wrap
+def cache_npsavez(f):
+ return _get_npsavez_wrap(f,False)
+
def cache_npsavez_compressed(f):
- def wrap(filename,*args,**kwargs):
- np_filename = os.path.splitext(filename)[0] + ".npy.npz"
- def loadsave():
- res = f(filename,*args,**kwargs)
- np.savez_compressed(np_filename,*res)
- return res
- if os.path.isfile(np_filename) and (not os.path.isfile(filename) or os.path.getmtime(np_filename) > os.path.getmtime(filename)):
- try:
- return [f['arr_%d'%i] for i in range(len(f.files()))]
- except:
- return loadsave()
- else:
- return loadsave()
- return wrap
\ No newline at end of file
+ return _get_npsavez_wrap(f, True)
\ No newline at end of file
diff --git a/wetb/utils/tests/test_caching.py b/wetb/utils/tests/test_caching.py
index bf308e76..816d3ec1 100644
--- a/wetb/utils/tests/test_caching.py
+++ b/wetb/utils/tests/test_caching.py
@@ -136,28 +136,32 @@ class TestCacheProperty(unittest.TestCase):
os.remove(tfp+'test.npy')
def test_cache_savez(self):
- if os.path.isfile(tfp+"test.npy.npy"):
- os.remove(tfp+'test.npy.npy')
- A = open_csv2(tfp + "test.csv")
- self.assertTrue(os.path.isfile(tfp+"test.npy.npz"))
+ npfilename = tfp+"test.npy.npz"
+ func = open_csv2
+ if os.path.isfile(npfilename):
+ os.remove(npfilename)
+ A = func(tfp + "test.csv")
+ self.assertTrue(os.path.isfile(npfilename))
np.testing.assert_array_equal(A[0],np.loadtxt(tfp + "test.csv"))
A[0][0]=-1
- np.save(tfp+"test.npy",A)
- B = open_csv(tfp + "test.csv")
+ np.savez(npfilename,A[0],A[1])
+ B = func(tfp + "test.csv")
np.testing.assert_array_equal(A,B)
- os.remove(tfp+'test.npy')
+ os.remove(npfilename)
def test_cache_savez_compressed(self):
- if os.path.isfile(tfp+"test2.npy.npy"):
- os.remove(tfp+'test2.npy.npy')
- A = open_csv2(tfp + "test2.csv")
- self.assertTrue(os.path.isfile(tfp+"test2.npy.npz"))
- np.testing.assert_array_equal(A[0],np.loadtxt(tfp + "test2.csv"))
+ npfilename = tfp+"test.npy.npzc"
+ func = open_csv3
+ if os.path.isfile(npfilename):
+ os.remove(npfilename)
+ A = func(tfp + "test.csv")
+ self.assertTrue(os.path.isfile(npfilename))
+ np.testing.assert_array_equal(A[0],np.loadtxt(tfp + "test.csv"))
A[0][0]=-1
- np.save(tfp+"test2.npy",A)
- B = open_csv(tfp + "test2.csv")
+ np.savez(npfilename,A[0],A[1])
+ B = func(tfp + "test.csv")
np.testing.assert_array_equal(A,B)
- os.remove(tfp+'test2.npy')
+ os.remove(npfilename)
if __name__ == "__main__":
#import sys;sys.argv = ['', 'Test.testName']
diff --git a/wetb/utils/tests/test_files/test2.csv b/wetb/utils/tests/test_files/test2.csv
deleted file mode 100644
index 158e1a69..00000000
--- a/wetb/utils/tests/test_files/test2.csv
+++ /dev/null
@@ -1,5 +0,0 @@
-0.000000000000000000e+00 5.000000000000000000e+00
-1.000000000000000000e+00 6.000000000000000000e+00
-2.000000000000000000e+00 7.000000000000000000e+00
-3.000000000000000000e+00 8.000000000000000000e+00
-4.000000000000000000e+00 9.000000000000000000e+00
--
GitLab
From 5da20b4cc4993288771fb0e3646ad61e3d704edd Mon Sep 17 00:00:00 2001
From: "Mads M. Pedersen" <mmpe@dtu.dk>
Date: Thu, 2 Nov 2017 09:45:26 +0100
Subject: [PATCH 3/3] fixed caching test
---
wetb/utils/caching.py | 2 +-
wetb/utils/tests/test_caching.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/wetb/utils/caching.py b/wetb/utils/caching.py
index df0c369f..b4fd8484 100644
--- a/wetb/utils/caching.py
+++ b/wetb/utils/caching.py
@@ -130,7 +130,7 @@ def cache_npsave(f):
def _get_npsavez_wrap(f, compress):
def wrap(filename,*args,**kwargs):
- np_filename = os.path.splitext(filename)[0] + ".npy.npz"+("","c")[compress]
+ np_filename = os.path.splitext(filename)[0] + ".npy%s.npz"%("",".c")[compress]
def loadsave():
res = f(filename,*args,**kwargs)
if compress:
diff --git a/wetb/utils/tests/test_caching.py b/wetb/utils/tests/test_caching.py
index 816d3ec1..483eaa12 100644
--- a/wetb/utils/tests/test_caching.py
+++ b/wetb/utils/tests/test_caching.py
@@ -150,7 +150,7 @@ class TestCacheProperty(unittest.TestCase):
os.remove(npfilename)
def test_cache_savez_compressed(self):
- npfilename = tfp+"test.npy.npzc"
+ npfilename = tfp+"test.npy.c.npz"
func = open_csv3
if os.path.isfile(npfilename):
os.remove(npfilename)
--
GitLab