Merge branch 'master' of gitlab.windenergy.dtu.dk:toolbox/WindEnergyToolbox

wetb/hawc2/htc_contents.py

@@ -218,6 +218,8 @@ class HTCOutputSection(HTCSection):
 
 
     def line_from_line(self, lines):
+        while lines[0].strip() == "":
+            lines.pop(0)
         name = lines[0].split()[0].strip()
         if name in ['filename', 'data_format', 'buffer', 'time']:
             return HTCLine.from_lines(lines)
@@ -242,6 +244,8 @@ class HTCOutputAtTimeSection(HTCOutputSection):
     type = None
     time = None
     def __init__(self, name, begin_comments="", end_comments=""):
+        if len(name.split()) < 3:
+            raise ValueError('"keyword" and "time" arguments required for output_at_time command:\n%s' % name)
         name, self.type, time = name.split()
         self.time = float(time)
         HTCOutputSection.__init__(self, name, begin_comments=begin_comments, end_comments=end_comments)
@@ -270,9 +274,11 @@ class HTCSensor(HTCLine):
         if len(line.split()) > 2:
             _3to2list5 = list(line.split())
             type, sensor, values, = _3to2list5[:2] + [_3to2list5[2:]]
-        else:
+        elif len(line.split()) == 2:
             type, sensor = line.split()
             values = []
+        else:
+            type, sensor, values = "", "", []
         def fmt(v):
             try:
                 if int(float(v)) == float(v):

wetb/hawc2/htc_file.py

@@ -14,6 +14,7 @@ from io import open
 from builtins import str
 from future import standard_library
 from wetb.utils.process_exec import pexec
+
 standard_library.install_aliases()
 from collections import OrderedDict
 
@@ -210,7 +211,14 @@ class HTCFile(HTCContents, HTCDefaults):
         hawc2exe = exe
         errorcode, stdout, stderr, cmd = pexec([hawc2exe, htcfile], self.modelpath)
 
-        if errorcode or 'Elapsed time' not in stderr:
+        if "logfile" in self.simulation:
+            with open(os.path.join(self.modelpath, self.simulation.logfile[0])) as fid:
+                log = fid.read()
+        else:
+            log = stderr
+
+
+        if errorcode or 'Elapsed time' not in log:
             raise Exception (str(stdout) + str(stderr))
 
 if "__main__" == __name__:

wetb/hawc2/log_file.py

@@ -150,8 +150,11 @@ class LogFile(LogInterpreter):
         # exist_ok does not exist in Python27
         if not os.path.exists(os.path.dirname(self.filename)):
             os.makedirs(os.path.dirname(self.filename))  #, exist_ok=True)
-        with open(self.filename, 'w', encoding='utf-8'):
-            pass
+        try:
+            with open(self.filename, 'w', encoding='utf-8'):
+                pass
+        except PermissionError as e:
+            raise PermissionError(str(e) + "\nLog file cannot be cleared. Check if it is open in another program")
         LogInterpreter.clear(self)
 
     def update_status(self):

wetb/hawc2/simulation.py

@@ -79,6 +79,7 @@ class Simulation(object):
     status = QUEUED
     def __init__(self, modelpath, htcfilename, hawc2exe="HAWC2MB.exe", copy_turbulence=True):
         self.modelpath = os.path.abspath(modelpath) + "/"
+        self.tmp_modelpath = self.modelpath
         self.folder = os.path.dirname(htcfilename)
         if not os.path.isabs(htcfilename):
             htcfilename = os.path.join(modelpath, htcfilename)
@@ -106,7 +107,8 @@ class Simulation(object):
         self.non_blocking_simulation_thread = Thread(target=self.simulate_distributed)
         self.updateStatusThread = UpdateStatusThread(self)
         self.host = LocalSimulationHost(self)
-
+        self.stdout = ""
+        self.returncode = 0
 
     def start(self, update_interval=1):
         """Start non blocking distributed simulation"""
@@ -207,7 +209,7 @@ class Simulation(object):
         self.update_status()
         self.is_simulating = False
         if self.host.returncode or self.errors:
-            raise Exception("Simulation error:\n" + "\n".join(self.errors))
+            raise Exception("Simulation error:\nReturn code: %d\n%s" % (self.host.returncode, "\n".join(self.errors)))
         elif self.logFile.status != log_file.DONE or self.logFile.errors:
             raise Warning("Simulation succeded with errors:\nLog status:%s\nErrors:\n%s" % (self.logFile.status, "\n".join(self.logFile.errors)))
         else:
@@ -325,7 +327,7 @@ class Simulation(object):
     def show_message(self, msg, title="Information"):
         print (msg)
 
-    def set_id(self):
+    def set_id(self, *args, **kwargs):
         pass
 
 
@@ -401,6 +403,7 @@ class LocalSimulationHost(SimulationResource):
         #must be called through simulation object
         self.returncode, self.stdout = 1, "Simulation failed"
         self.simulationThread.start()
+        self.sim.set_id(self.sim.simulation_id, "Localhost(pid:%d)" % self.simulationThread.process.pid, self.tmp_modelpath)
         self.simulationThread.join()
         self.returncode, self.stdout = self.simulationThread.res
         self.logFile.update_status()

wetb/utils/cluster_tools/cluster_resource.py

@@ -24,7 +24,10 @@ class Resource(object):
 
     def ok2submit(self):
         """Always ok to have min_cpu cpus and ok to have more if there are min_free free cpus"""
-        total, free, user = self.check_resources()
+        try:
+            total, free, user = self.check_resources()
+        except:
+            return False
 
         if user < self.min_cpu:
             return True
@@ -75,9 +78,7 @@ class SSHPBSClusterResource(Resource, SSHClient):
                 cpu_free, nodeSum = pbswrap.count_cpus(users, host, pbsnodes)
 
                 return nodeSum['used_cpu'] + cpu_free, cpu_free, cpu_user
-            except IOError as e:
-                raise e
-            except:
+            except Exception as e:
                 raise EnvironmentError("check resources failed")
 
     def jobids(self, jobname_prefix):

wetb/utils/cluster_tools/ssh_client.py

@@ -25,6 +25,8 @@ class SSHClient(object):
         self.key = key
         self.disconnect = 0
         self.client = None
+        self.sftp = None
+        self.transport = None
         if key is not None:
             self.key = paramiko.RSAKey.from_private_key(StringIO(key), password=passphrase)
 
@@ -33,7 +35,8 @@ class SSHClient(object):
 
     def __enter__(self):
         self.disconnect += 1
-        if self.client is None:
+        if self.client is None or self.client._transport is None or self.client._transport.is_active() is False:
+            self.close()
             self.connect()
         return self.client
 
@@ -44,9 +47,7 @@ class SSHClient(object):
         self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
         self.client.connect(self.host, self.port, username=self.username, password=self.password, pkey=self.key, timeout=self.TIMEOUT)
         assert self.client is not None
-        self.transport = paramiko.Transport((self.host, self.port))
-        self.transport.connect(username=self.username, password=self.password)
-        self.sftp = paramiko.SFTPClient.from_transport(self.transport)
+        self.sftp = paramiko.SFTPClient.from_transport(self.client._transport)
         return self
 
     def __exit__(self, *args):
@@ -85,11 +86,12 @@ class SSHClient(object):
             print (ret)
 
     def close(self):
-        if self.client is not None:
-            self.client.close()
-            self.client = None
-        self.sftp.close()
-        self.transport.close()
+        for x in ["sftp", "client" ]:
+            try:
+                getattr(self, x).close()
+                setattr(self, x, None)
+            except:
+                pass
         self.disconnect = False
 
     def file_exists(self, filename):
@@ -180,6 +182,7 @@ class SharedSSHClient(SSHClient):
 
         while self.next != threading.currentThread():
             time.sleep(1)
+        SSHClient.__enter__(self)
         return self.client
 
     def __exit__(self, *args):

wetb/utils/geometry.py

@@ -168,9 +168,9 @@ def abvrel2xyz(alpha, beta, vrel):
     Parameters
     ----------
     alpha : array_like
-        Pitot tube angle of attack. Zero: Parallel to pitot tube. Positive: Flow from wind side (pressure side)
+        Pitot tube angle of attack [rad]. Zero: Parallel to pitot tube. Positive: Flow from wind side (pressure side)
     beta : array_like
-        Pitot tube side slip angle. Zero: Parallel to pitot tube. Positive: Flow from root side
+        Pitot tube side slip angle [rad]. Zero: Parallel to pitot tube. Positive: Flow from root side
     vrel : array_like
         Pitot tube relative velocity. Positive: flow towards pitot tube