diff --git a/wetb/hawc2/htc_file.py b/wetb/hawc2/htc_file.py
index 56adcc3683b2f3373859ee79113e2013d80cbada..9c4761b6bda21d6343a58dd28fcfab3e80e09018 100644
--- a/wetb/hawc2/htc_file.py
+++ b/wetb/hawc2/htc_file.py
@@ -13,7 +13,8 @@ from __future__ import absolute_import
from io import open
from builtins import str
from future import standard_library
-from wetb.utils.process_exec import pexec, unix_filename
+from wetb.utils.process_exec import pexec
+from wetb.utils.cluster_tools.cluster_resource import unix_path
standard_library.install_aliases()
from collections import OrderedDict
@@ -23,6 +24,9 @@ import os
from copy import copy
+def fmt_path(path):
+ return path.lower().replace("\\","/")
+
class HTCFile(HTCContents, HTCDefaults):
"""Wrapper for HTC files
@@ -39,13 +43,13 @@ class HTCFile(HTCContents, HTCDefaults):
modelpath = "../"
initial_comments = None
_contents = None
- def __init__(self, filename=None, modelpath="../"):
+ def __init__(self, filename=None, relative_modelpath="../"):
if filename is not None:
- self.modelpath = os.path.realpath(os.path.join(os.path.dirname(filename), modelpath))
+ self.modelpath = os.path.realpath(os.path.join(os.path.dirname(filename), relative_modelpath))
self.filename = filename
else:
- self.modelpath = modelpath
+ self.modelpath = relative_modelpath
@@ -92,7 +96,7 @@ class HTCFile(HTCContents, HTCDefaults):
def readfilelines(self, filename):
- with open(unix_filename(filename), encoding='cp1252') as fid:
+ with open(unix_path(filename), encoding='cp1252') as fid:
lines = list(fid.readlines())
if lines[0].encode().startswith(b'\xc3\xaf\xc2\xbb\xc2\xbf'):
lines[0] = lines[0][3:]
diff --git a/wetb/hawc2/simulation.py b/wetb/hawc2/simulation.py
index f54b26e56d8eaff5387780cb1d22edf09939bfb0..8e96c0483b03abed505b78c81ddb231b5d8ae970 100755
--- a/wetb/hawc2/simulation.py
+++ b/wetb/hawc2/simulation.py
@@ -18,9 +18,10 @@ import time
from future import standard_library
from wetb.hawc2 import log_file
-from wetb.hawc2.htc_file import HTCFile
+from wetb.hawc2.htc_file import HTCFile, fmt_path
from wetb.hawc2.log_file import LogFile
-from wetb.utils.cluster_tools.cluster_resource import unix_path
+
+
standard_library.install_aliases()
@@ -75,7 +76,7 @@ class Simulation(object):
def __init__(self, modelpath, htcfilename, hawc2exe="HAWC2MB.exe", copy_turbulence=True):
self.modelpath = os.path.abspath(modelpath) + "/"
if os.path.isabs(htcfilename):
- htcfilename = os.path.relpath(htcfilename, modelpath).replace("\\","/")
+ htcfilename = os.path.relpath(htcfilename, modelpath)
if htcfilename.startswith("input/"):
htcfilename=htcfilename[6:]
exists = [os.path.isfile(os.path.join(modelpath, htcfilename)),
@@ -91,22 +92,22 @@ class Simulation(object):
self.exepath = self.modelpath + "input/"
else:
self.exepath = self.modelpath
+ htcfilename = fmt_path(htcfilename)
self.tmp_modelpath = self.exepath
self.folder = os.path.dirname(htcfilename)
- if not os.path.isabs(htcfilename):
- htcfilename = os.path.join(self.exepath, htcfilename)
+
self.filename = os.path.basename(htcfilename)
- self.htcFile = HTCFile(htcfilename, self.exepath)
+ self.htcFile = HTCFile(os.path.join(self.exepath, htcfilename), self.exepath)
self.time_stop = self.htcFile.simulation.time_stop[0]
self.hawc2exe = hawc2exe
self.copy_turbulence = copy_turbulence
- self.simulation_id = unix_path(os.path.relpath(htcfilename, self.modelpath) + "_%d" % id(self)).replace("/", "_")
+ self.simulation_id = (htcfilename + "_%d" % id(self)).replace("\\","/").replace("/", "_")
if self.simulation_id.startswith("input_"):
self.simulation_id = self.simulation_id[6:]
- self.stdout_filename = os.path.splitext(unix_path(os.path.relpath(htcfilename, self.modelpath)).replace('htc', 'stdout', 1))[0] + ".out"
+ self.stdout_filename = fmt_path(os.path.join(os.path.relpath(self.exepath, self.modelpath),
+ (os.path.splitext(htcfilename)[0] + ".out").replace('htc', 'stdout', 1)))
if self.ios:
-
assert self.stdout_filename.startswith("input/")
self.stdout_filename = self.stdout_filename.replace("input/", "../output/")
#self.stdout_filename = "stdout/%s.out" % self.simulation_id
@@ -118,7 +119,7 @@ class Simulation(object):
self.log_filename = os.path.relpath(self.log_filename, self.modelpath)
else:
self.log_filename = os.path.relpath(self.log_filename)
- self.log_filename = unix_path(self.log_filename)
+ self.log_filename = fmt_path(self.log_filename)
self.logFile = LogFile(os.path.join(self.exepath, self.log_filename), self.time_stop)
self.logFile.clear()
self.last_status = self.status
@@ -257,19 +258,20 @@ class Simulation(object):
dst = os.path.relpath(os.path.abspath(dst), self.exepath)
else:
dst = os.path.relpath (dst)
- dst = unix_path(dst)
+ dst = fmt_path(dst)
assert not os.path.relpath(os.path.join(self.exepath, dst), self.modelpath).startswith(".."), "%s referes to a file outside the model path\nAll input files be inside model path" % dst
return dst
+ turb_files = [f for f in self.htcFile.turbulence_files() if self.copy_turbulence and not os.path.isfile(os.path.join(self.exepath, f))]
if self.ios:
output_patterns = [fmt(dst) for dst in (["../output/*", "../output/"] +
- ([], self.htcFile.turbulence_files())[self.copy_turbulence] +
+ turb_files +
[os.path.join(self.exepath, self.stdout_filename)])]
- output_files = set([f for pattern in output_patterns for f in self.host.glob(unix_path(os.path.join(self.tmp_exepath, pattern)), recursive=True)])
+ output_files = set([fmt_path(f) for pattern in output_patterns for f in self.host.glob(fmt_path(os.path.join(self.tmp_exepath, pattern)), recursive=True)])
else:
output_patterns = [fmt(dst) for dst in (self.htcFile.output_files() +
- ([], self.htcFile.turbulence_files())[self.copy_turbulence] +
+ turb_files +
[os.path.join(self.exepath, self.stdout_filename)])]
- output_files = set([f for pattern in output_patterns for f in self.host.glob(unix_path(os.path.join(self.tmp_exepath, pattern)))])
+ output_files = set([fmt_path(f) for pattern in output_patterns for f in self.host.glob(fmt_path(os.path.join(self.tmp_exepath, pattern)))])
try:
self.host._finish_simulation(output_files)
if self.status != ERROR:
@@ -349,7 +351,7 @@ class Simulation(object):
def fix_errors(self):
def confirm_add_additional_file(folder, file):
if os.path.isfile(os.path.join(self.modelpath, folder, file)):
- filename = unix_path(os.path.join(folder, file))
+ filename = fmt_path(os.path.join(folder, file))
if self.get_confirmation("File missing", "'%s' seems to be missing in the temporary working directory. \n\nDo you want to add it to additional_files.txt" % filename):
self.add_additional_input_file(filename)
self.show_message("'%s' is now added to additional_files.txt.\n\nPlease restart the simulation" % filename)
diff --git a/wetb/hawc2/simulation_resources.py b/wetb/hawc2/simulation_resources.py
index 973cb1204cb42d2af39aadaf39d16f8b7518ad39..7ee64a45411e53e32f8a76e1aa17cfe9951c4c42 100644
--- a/wetb/hawc2/simulation_resources.py
+++ b/wetb/hawc2/simulation_resources.py
@@ -22,6 +22,7 @@ from wetb.utils.cluster_tools.cluster_resource import LocalResource, \
from wetb.utils.cluster_tools.pbsjob import SSHPBSJob, NOT_SUBMITTED, DONE
from wetb.utils.cluster_tools.ssh_client import SSHClient
from wetb.utils.timing import print_time
+from wetb.hawc2.htc_file import fmt_path
class SimulationHost(object):
@@ -116,7 +117,16 @@ class LocalSimulationHost(SimulationHost):
try:
shutil.rmtree(self.tmp_modelpath, ignore_errors=False)
except (PermissionError, OSError) as e:
- raise Warning("Fail to remove temporary files and folders on %s\n%s"%(self.resource.host, str(e)))
+ try:
+ #change permissions and delete
+ for root, folders, files in os.walk(self.tmp_modelpath):
+ for folder in folders:
+ os.chmod(os.path.join(root, folder), 0o666)
+ for file in files:
+ os.chmod(os.path.join(root, file), 0o666)
+ shutil.rmtree(self.tmp_modelpath)
+ except (PermissionError, OSError) as e:
+ raise Warning("Fail to remove temporary files and folders on %s\n%s"%(self.resource.host, str(e)))
def update_logFile_status(self):
@@ -124,7 +134,9 @@ class LocalSimulationHost(SimulationHost):
def stop(self):
self.simulationThread.stop()
- self.simulationThread.join()
+ if self.simulationThread.is_alive():
+ self.simulationThread.join()
+ print ("simulatino_resources.stop joined")
@@ -183,7 +195,10 @@ class SimulationThread(Thread):
def stop(self):
if hasattr(self, 'process'):
- subprocess.Popen("TASKKILL /F /PID {pid} /T".format(pid=self.process.pid))
+ try:
+ subprocess.Popen("TASKKILL /F /PID {pid} /T".format(pid=self.process.pid))
+ except:
+ pass
class PBSClusterSimulationResource(SSHPBSClusterResource):
@@ -246,7 +261,7 @@ class PBSClusterSimulationHost(SimulationHost, SSHClient):
if v == 0:
return datetime.strptime(out.strip(), "%Y,%m,%d,%H,%M,%S")
- @print_time
+ #@print_time
def _prepare_simulation(self, input_files):
with self:
self.execute(["mkdir -p .hawc2launcher/%s" % self.simulation_id], verbose=False)
@@ -268,7 +283,7 @@ class PBSClusterSimulationHost(SimulationHost, SSHClient):
- @print_time
+ #@print_time
def _finish_simulation(self, output_files):
with self:
download_failed = []
@@ -377,10 +392,10 @@ class PBSClusterSimulationHost(SimulationHost, SSHClient):
def pbsjobfile(self, ios=False):
cp_back = ""
- for folder in set([unix_path(os.path.relpath(os.path.dirname(f))) for f in self.htcFile.output_files() + self.htcFile.turbulence_files()]):
+ for folder in set([fmt_path(os.path.relpath(os.path.dirname(f))) for f in self.htcFile.output_files() + self.htcFile.turbulence_files()]):
cp_back += "mkdir -p $PBS_O_WORKDIR/%s/. \n" % folder
cp_back += "cp -R -f %s/. $PBS_O_WORKDIR/%s/.\n" % (folder, folder)
- rel_htcfilename = unix_path(os.path.relpath(self.htcFile.filename, self.exepath))
+ rel_htcfilename = fmt_path(os.path.relpath(self.htcFile.filename, self.exepath))
init="""
### Standard Output
diff --git a/wetb/hawc2/tests/test_htc_file.py b/wetb/hawc2/tests/test_htc_file.py
index 273bbd15851d3600d929ad6550d509dbcbbb8715..ff649e749ae467ec643a7d1353021f1eb8172d93 100644
--- a/wetb/hawc2/tests/test_htc_file.py
+++ b/wetb/hawc2/tests/test_htc_file.py
@@ -250,10 +250,6 @@ class TestHtcFile(unittest.TestCase):
def test_htc_reset(self):
htcfile = HTCFile(self.testfilepath + "test.htc")
self.assertEqual(htcfile.wind.wsp[0], 10)
-
-
-
-
if __name__ == "__main__":
#import sys;sys.argv = ['', 'Test.testName']
diff --git a/wetb/utils/cluster_tools/cluster_resource.py b/wetb/utils/cluster_tools/cluster_resource.py
index 7e657d1b2aa4a65156d6c463d859c435c7c1da00..aea4f8df24ee74610926f0ea66e8d19b8c14857e 100644
--- a/wetb/utils/cluster_tools/cluster_resource.py
+++ b/wetb/utils/cluster_tools/cluster_resource.py
@@ -3,16 +3,60 @@ Created on 04/04/2016
@author: MMPE
'''
+import glob
import multiprocessing
+import os
+import re
import threading
-
-
from wetb.utils.cluster_tools import pbswrap
from wetb.utils.cluster_tools.ssh_client import SSHClient, SharedSSHClient
-def unix_path(path):
- return path.replace("\\", "/").lower()
+
+def unix_path(path, cwd=None, fail_on_missing=False):
+ """Convert case insensitive filename into unix case sensitive filename
+
+ If no matching file or folder is found an error is raised
+
+ Parameters
+ ---------
+ x : str
+ Case insensitive filename or folder
+
+ Returns
+ -------
+ Filename or folder name
+
+ """
+ if cwd:
+ path = os.path.join(cwd, path)
+ path = os.path.abspath(path)
+ r = glob.glob(re.sub(r'([^:/\\])(?=[/\\]|$)', r'[\1]', path))
+ if r:
+ path = r[0]
+ elif fail_on_missing:
+ raise FileExistsError("File or folder matching '%s' not found"%path)
+ if cwd:
+ path = os.path.relpath(path, cwd)
+ if os.path.isdir(path):
+ path+="/"
+ return path.replace("\\","/")
+
+# filename = os.path.realpath(filename.replace("\\", "/")).replace("\\", "/")
+# ufn, rest = os.path.splitdrive(filename)
+# ufn += "/"
+# for f in rest[1:].split("/"):
+# f_lst = [f_ for f_ in os.listdir(ufn) if f_.lower() == f.lower()]
+# if len(f_lst) > 1:
+# f_lst = [f_ for f_ in f_lst if f_ == f]
+# elif len(f_lst) == 0:
+# raise IOError("'%s' not found in '%s'" % (f, ufn))
+# else: # one match found
+# ufn = os.path.join(ufn, f_lst[0])
+# return ufn.replace("\\", "/")
+
+
+
class Resource(object):
diff --git a/wetb/utils/cluster_tools/ssh_client.py b/wetb/utils/cluster_tools/ssh_client.py
index d3480e7fbddbebcac7ed7aefc8b0f00052cb3557..bd5450dc29efe175eed049422e73d52dc5442e99 100644
--- a/wetb/utils/cluster_tools/ssh_client.py
+++ b/wetb/utils/cluster_tools/ssh_client.py
@@ -106,17 +106,22 @@ class SSHClient(object):
files = set([os.path.abspath(f) for f in files])
compression_levels = {0:zipfile.ZIP_STORED, 1:zipfile.ZIP_DEFLATED, 2:zipfile.ZIP_BZIP2, 3:zipfile.ZIP_LZMA}
- zn = 'tmp.zip'
+ zn = 'tmp_%s_%s.zip'%(id(self),time.time())
zipf = zipfile.ZipFile(zn, 'w', compression_levels[compression_level])
- for f in files:
- zipf.write(f, os.path.relpath(f, localpath))
- zipf.close()
- remote_zn = os.path.join(remotepath, zn).replace("\\","/")
- self.execute("mkdir -p %s"%(remotepath))
+ try:
+ for f in files:
+ zipf.write(f, os.path.relpath(f, localpath))
+ zipf.close()
+ remote_zn = os.path.join(remotepath, zn).replace("\\","/")
+ self.execute("mkdir -p %s"%(remotepath))
+
+ self.upload(zn, remote_zn)
+ self.execute("unzip %s -d %s && rm %s"%(remote_zn, remotepath, remote_zn))
+ except:
+ raise
+ finally:
+ os.remove(zn)
- self.upload(zn, remote_zn)
- os.remove(zn)
- self.execute("unzip %s -d %s && rm %s"%(remote_zn, remotepath, remote_zn))
def download_files(self, remote_path, localpath, file_lst=["."], compression_level=1):
if not isinstance(file_lst, (tuple, list)):
diff --git a/wetb/utils/process_exec.py b/wetb/utils/process_exec.py
index 9782c2552551b9019021295b920e71d5953e2560..eeaf03497d7ad6d738ba1d96e49990c14d9a8c9b 100644
--- a/wetb/utils/process_exec.py
+++ b/wetb/utils/process_exec.py
@@ -10,6 +10,8 @@ from __future__ import absolute_import
from builtins import range
from builtins import str
from future import standard_library
+import glob
+import re
standard_library.install_aliases()
import os
@@ -59,32 +61,3 @@ def exec_process(process):
return errorcode, stdout.decode(), stderr.decode()
-def unix_filename(filename):
- """Convert case insensitive filename into unix case sensitive filename
-
- If more than one case insensitive matching file or folder is found, case sensitive matching is used
-
- Parameters
- ---------
- x : str
- Case insensitive filename
-
- Returns
- -------
- Filename
-
- """
- filename = os.path.realpath(filename.replace("\\", "/")).replace("\\", "/")
- ufn, rest = os.path.splitdrive(filename)
- ufn += "/"
- for f in rest[1:].split("/"):
- f_lst = [f_ for f_ in os.listdir(ufn) if f_.lower() == f.lower()]
- if len(f_lst) > 1:
- f_lst = [f_ for f_ in f_lst if f_ == f]
- elif len(f_lst) == 0:
- raise IOError("'%s' not found in '%s'" % (f, ufn))
- else: # one match found
- ufn = os.path.join(ufn, f_lst[0])
- return ufn.replace("\\", "/")
-
-
diff --git a/wetb/utils/tests/test_geometry.py b/wetb/utils/tests/test_geometry.py
index a624ce78f00ed2fccc5a05649ae16ac3a4740c85..581dc34911335b5facd483cfd79b8cc2e739361c 100644
--- a/wetb/utils/tests/test_geometry.py
+++ b/wetb/utils/tests/test_geometry.py
@@ -11,7 +11,7 @@ from future import standard_library
standard_library.install_aliases()
import unittest
-import wetb.gtsdf
+
import numpy as np
from wetb.utils.geometry import rad, deg, mean_deg, sind, cosd, std_deg, tand
import os
diff --git a/wetb/utils/tests/test_process_exe.py b/wetb/utils/tests/test_process_exe.py
index 32e258cf50a112a9b45f53b357cffb3c81f0a263..78583f9febd98dd482f01490fd26af2772bb8678 100644
--- a/wetb/utils/tests/test_process_exe.py
+++ b/wetb/utils/tests/test_process_exe.py
@@ -4,17 +4,12 @@ Created on 11/07/2016
@author: MMPE
'''
import unittest
-from wetb.utils.process_exec import unix_filename
import os
-
+tfp = os.path.join(os.path.dirname(__file__), 'test_files/')
class Test(unittest.TestCase):
-
-
- def testUnix_filename(self):
- ufn = "WindEnergyToolbox/wetb/hawc2/Hawc2io.py"
- f = os.path.join(os.path.dirname(__file__), r"../../../..\windenergytoolbox/wetb/HAWC2/hawc2io.py")
- self.assertEqual(unix_filename(f)[-len(ufn):], ufn)
+ pass
+
if __name__ == "__main__":