From 04edd3759bcecf3e2541c26fac597f749933a39d Mon Sep 17 00:00:00 2001
From: mmpe <mmpe@dtu.dk>
Date: Mon, 8 Oct 2018 09:46:07 +0200
Subject: [PATCH] Fix tests
---
wetb/prepost/Simulations.py | 6 +++++
wetb/prepost/simchunks.py | 12 ++++++---
wetb/prepost/tests/test_Simulations.py | 34 +++++++++++++++++++++++++-
wetb/utils/test_files.py | 17 +++++++++----
wetb/utils/tests/test_test_files.py | 6 ++++-
5 files changed, 65 insertions(+), 10 deletions(-)
diff --git a/wetb/prepost/Simulations.py b/wetb/prepost/Simulations.py
index c754717..c3fe5ae 100755
--- a/wetb/prepost/Simulations.py
+++ b/wetb/prepost/Simulations.py
@@ -56,6 +56,12 @@ from wetb.prepost import prepost
from wetb.dlc import high_level as dlc
from wetb.prepost.GenerateHydro import hydro_input
from wetb.utils.envelope import compute_envelope
+from os.path import join as os_path_join
+
+def join_path(*args):
+ return os_path_join(*args).replace("\\","/")
+os.path.join = join_path
+
def load_pickled_file(source):
FILE = open(source, 'rb')
diff --git a/wetb/prepost/simchunks.py b/wetb/prepost/simchunks.py
index 785e9dd..abb00f7 100644
--- a/wetb/prepost/simchunks.py
+++ b/wetb/prepost/simchunks.py
@@ -147,7 +147,7 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
pbs_tmplate += "#PBS -q [queue]\n"
pbs_tmplate += "\n"
- # this causes troubles on CI runner for the tests (line endings?)
+ # FIXME: this causes troubles on CI runner for the tests (line endings?)
# pbs_tmplate = """
# ### Standard Output
# #PBS -N [job_name]
@@ -299,11 +299,17 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
for db, base_name in zip(turb_db_tags, base_name_tags):
turb_db_dirs = df[db] + df[base_name]
# When set to None, the DataFrame will have text as None
- # FIXME: CI runner has and old pandas version
+ # FIXME: CI runner has and old pandas version (v0.14.1)
try:
turb_db_src = turb_db_dirs[turb_db_dirs.str.find('None')==-1]
except AttributeError:
- turb_db_src = turb_db_dirs[turb_db_dirs.str.findall('None')==-1]
+ # and findall returns list with the search str occuring as
+ # many times as found in the str...
+ # sel should be True if str does NOT occur in turb_db_dirs
+ # meaning if findall returns empty list
+ findall = turb_db_dirs.str.findall('None').tolist()
+ sel = [True if len(k)==0 else False for k in findall]
+ turb_db_src = turb_db_dirs[sel]
pbs += '\n'
pbs += '# copy to scratch db directory for %s, %s\n' % (db, base_name)
for k in turb_db_src.unique():
diff --git a/wetb/prepost/tests/test_Simulations.py b/wetb/prepost/tests/test_Simulations.py
index 8af4bec..4f18d65 100644
--- a/wetb/prepost/tests/test_Simulations.py
+++ b/wetb/prepost/tests/test_Simulations.py
@@ -68,7 +68,38 @@ class TestGenerateInputs(Template):
postpro_node_zipchunks=False,
postpro_node=False)
- # we can not check-in empty dirs so we can not compare the complete
+ def cmp_dir(dir1, dir2):
+ lst1, lst2 = map(os.listdir, (dir1, dir2))
+ self.assertEqual(";".join(lst1), ";".join(lst2))
+ for f1, f2 in zip(lst1, lst2):
+ if f1.endswith(".zip") or f1.endswith(".xlsx"):
+ continue
+ if os.path.isdir(os.path.join(dir1, f1)):
+ cmp_dir(os.path.join(dir1, f1), os.path.join(dir2, f2))
+ else:
+ try:
+ with open(os.path.join(dir1, f1)) as fid1:
+ l1 = fid1.readlines()
+ with open(os.path.join(dir2, f2)) as fid2:
+ l2 = fid2.readlines()
+
+ self.assertEqual(len(l1), len(l2))
+ self.assertTrue(all([l1_ == l2_ for l1_, l2_ in zip(l1, l2)]))
+ except:
+ print("=" * 30)
+ print(os.path.join(dir1, f1))
+ print(os.path.join(dir2, f2))
+ print(dir1[[d1 != d2 for d1, d2 in zip(dir1, dir2)].index(True):])
+ print(f1)
+ for i in range(len(l1)):
+ if l1[i] != l2[i]:
+ print("%03d, rem: %s" % (i, l1[i].strip()))
+ print("%03d, ref: %s" % (i, l2[i].strip()))
+ print()
+ raise
+
+
+ # we can not git check-in empty dirs so we can not compare the complete
# directory structure withouth manually creating the empty dirs here
for subdir in ['control', 'data', 'htc', 'pbs_in', 'pbs_in_turb',
'htc/_master', 'htc/dlc01_demos', 'pbs_in/dlc01_demos',
@@ -78,6 +109,7 @@ class TestGenerateInputs(Template):
# the zipfiles are taken care of separately
ignore = ['remote_chnk_00000.zip']
cmp = filecmp.dircmp(remote, ref, ignore=ignore)
+ cmp_dir(remote, ref)
self.assertEqual(len(cmp.diff_files), 0,
"{} {}".format(subdir, cmp.diff_files))
self.assertEqual(len(cmp.right_only), 0,
diff --git a/wetb/utils/test_files.py b/wetb/utils/test_files.py
index 6e230c3..e4e3e88 100644
--- a/wetb/utils/test_files.py
+++ b/wetb/utils/test_files.py
@@ -6,7 +6,11 @@ Created on 20. jul. 2017
import os
import wetb
import inspect
-wetb_rep_path = os.path.join(os.path.dirname(wetb.__file__), "../")
+from urllib.request import urlretrieve
+
+wetb_rep_path = os.path.abspath(os.path.dirname(wetb.__file__) + "/../") + "/"
+local_TestFiles_path = wetb_rep_path + "TestFiles/"
+remote_TestFiles_url = "https://gitlab.windenergy.dtu.dk/toolbox/TestFiles/raw/master/"
def _absolute_filename(filename):
@@ -19,10 +23,13 @@ def _absolute_filename(filename):
def get_test_file(filename):
filename = _absolute_filename(filename)
- if os.path.exists(filename):
- return filename
- else:
- return os.path.join(wetb_rep_path, 'TestFiles', os.path.relpath(filename, wetb_rep_path))
+ if not os.path.exists(filename):
+ rel_path = os.path.relpath(filename, wetb_rep_path).replace("\\","/")
+ filename = local_TestFiles_path + rel_path
+ if not os.path.exists(filename):
+ urlretrieve(remote_TestFiles_url + rel_path, filename)
+ return filename
+
diff --git a/wetb/utils/tests/test_test_files.py b/wetb/utils/tests/test_test_files.py
index 6cbe8f6..1b38c2d 100644
--- a/wetb/utils/tests/test_test_files.py
+++ b/wetb/utils/tests/test_test_files.py
@@ -4,7 +4,8 @@ Created on 20. jul. 2017
@author: mmpe
'''
import unittest
-from wetb.utils.test_files import move2test_files, get_test_file
+from wetb.utils.test_files import move2test_files, get_test_file,\
+ local_TestFiles_path
import os
from wetb.utils import test_files
import wetb
@@ -24,6 +25,9 @@ class Test_test_files(unittest.TestCase):
self.assertTrue(os.path.isfile(dst))
def test_test_files(self):
+ fn = local_TestFiles_path + "wetb/utils/tests/test_files/test_file.txt"
+ if os.path.isfile(fn):
+ os.remove(fn)
fn1 = get_test_file(tfp+'test_file.txt')
self.assertTrue(os.path.isfile(fn1))
fn2 = get_test_file('test_file.txt')
--
GitLab