Merge branch 'master' of gitlab.windenergy.dtu.dk:toolbox/WindEnergyToolbox

.gitlab-ci.yml

-before_script:
-  - apt-get update
-  # uncomment first time
-  #- rm -rf TestFiles
-  #- git submodule update --init
-  #- git submodule sync
-  #- git submodule update
-
+image: dtuwindenergy/wetb
 
 test-3.4:
-  image: mmpe/wetb
+  stage:  
+    test
+#  except:
+#    - test_pypi
   script:
-  #- python3 setup.py test
+  - apt-get update
   - pip3 install pytest
   - pip3 install mock
   - python3 -m pytest --cov=wetb
   tags:
   - python
+
+  # ===== BUILD WHEELS AND UPLOAD TO PYPI =====
+pypi_linux:
+  stage:  
+    deploy
+  only:
+    - tags
+    - test_pypi
+  script:
+    - apt-get update
+    - pip install -e . --upgrade
+    - python3 -c 'from git_utils import write_vers; write_vers()'
+    - python3 -m pip install -U setuptools wheel
+    - python3 setup.py sdist bdist_wheel
+    - python3 -m pip install -U twine
+    - python3 -c 'from git_utils import rename_dist_file; rename_dist_file()'
+    - twine upload dist/* -u $TWINE_USERNAME -p $TWINE_PASSWORD
+    #- twine upload --repository-url https://test.pypi.org/legacy/ dist/* -u $TWINE_USERNAME -p $TWINE_PASSWORD # for testing purposes
+  tags:  
+    - python
+
+
+pypi_windows:
+  stage:  
+    deploy
+  only:
+    - tags
+    - test_pypi
+  script:
+    - c:/Anaconda3/envs/WindEnergyToolbox/python.exe -c "from git_utils import write_vers; write_vers()"
+    - c:/Anaconda3/envs/WindEnergyToolbox/python.exe setup.py bdist_wheel
+    - twine upload dist/* -u %TWINE_USERNAME% -p %TWINE_PASSWORD%
+    #- twine upload --repository-url https://test.pypi.org/legacy/ dist/* -u %TWINE_USERNAME% -p %TWINE_PASSWORD% # for testing purposes
+  tags:  
+    - CPAV_old
+

.pypirc

+[distutils]
+index-servers=
+    pypi
+    #testpypi # include this line for testing
+
+[testpypi]
+repository: https://test.pypi.org/legacy/ # use this line when testing
+#repository: https://upload.pypi.org/legacy/
+username: DTUWindEnergy

docker/Dockerfile

+FROM continuumio/anaconda3:latest
+MAINTAINER Mikkel Friis-Møller <mikf@dtu.dk>
+
+RUN apt-get update && \
+    apt-get install make && \
+    apt-get install libgl1-mesa-glx -y && \
+    apt-get install gcc gfortran -y
+	
+RUN conda update -y conda && \
+    conda install -y sphinx_rtd_theme && \
+	conda install setuptools_scm future h5py pytables pytest pytest-cov nose sphinx blosc pbr paramiko && \
+	conda install scipy pandas matplotlib cython xlrd coverage xlwt openpyxl psutil pandoc twine pypandoc && \
+	conda install -c conda-forge pyscaffold sshtunnel --no-deps && \
+    conda clean -y --all
+	
+RUN pip install --upgrade pip && \
+	pip install --no-cache-dir git+https://gitlab.windenergy.dtu.dk/toolbox/WindEnergyToolbox.git

docker/Makefile

+# build image
+docker build -t dtuwindenergy/wetb ./
+# or pull from docker hub
+  
+# create container
+docker create -it --name wetb dtuwindenergy/wetb
+
+# start container
+docker start wetb
+
+# enter container with bash prompt
+docker exec -it wetb bash
+
+# push
+docker push dtuwindenergy/wetb
+
+#checkout single branch for testing:
+git clone --single-branch --branch test_pypi https://gitlab.windenergy.dtu.dk/toolbox/WindEnergyToolbox.git
+
+#Copy local folder into docker container:
+docker cp "C:\Sandbox\Git\WindEnergyToolbox\." wetb:/WETB/

docs/developer-guide.md

@@ -262,6 +262,14 @@ To be written
 
 ## Make and upload wheels to PyPi
 
+### Linux
+
+For Linux the wheels are now automatically build and pushed to pypi when creating a tag.
+
+### Windows
+
+For Windows the automated build and push to pypi has not yet been implemented and the developer has to follow the manual procedure outlined below:
+
 Workflow for creating and uploading wheels is as follows:
 
 - Make tag: ```git tag "vX.Y.Z"```, and push tag to remote: ```git push --tags```

git_utils.py

+'''
+Created on 28. jul. 2017
+
+@author: mmpe
+'''
+import os
+import subprocess
+
+
+def _run_git_cmd(cmd, git_repo_path=None):
+    git_repo_path = git_repo_path or os.getcwd()
+    if not os.path.isdir(os.path.join(git_repo_path, ".git")):
+        raise Warning("'%s' does not appear to be a Git repository." % git_repo_path)
+    try:
+        process = subprocess.Popen(cmd,
+                                   stdout=subprocess.PIPE,
+                                   universal_newlines=True,
+                                   cwd=os.path.abspath(git_repo_path))
+        stdout = process.communicate()[0]
+        if process.returncode != 0:
+            raise EnvironmentError()
+        return stdout.strip()
+
+    except EnvironmentError as e:
+        raise e
+        raise Warning("unable to run git")
+
+
+def get_git_version(git_repo_path=None):
+    cmd = ["git", "describe", "--tags", "--dirty", "--always"]
+    return _run_git_cmd(cmd, git_repo_path)
+
+
+def get_tag(git_repo_path=None):
+    return _run_git_cmd(['git', 'describe', '--tags', '--abbrev=0'], git_repo_path)
+
+
+def set_tag(tag, push, git_repo_path=None):
+    _run_git_cmd(["git", "tag", tag], git_repo_path)
+    if push:
+        _run_git_cmd(["git", "push"], git_repo_path)
+        _run_git_cmd(["git", "push", "--tags"], git_repo_path)
+
+
+def update_git_version(version_module, git_repo_path=None):
+    """Update <version_module>.__version__ to git version"""
+
+    version_str = get_git_version(git_repo_path)
+    assert os.path.isfile(version_module.__file__)
+    with open(version_module.__file__, "w") as fid:
+        fid.write("__version__ = '%s'" % version_str)
+
+    # ensure file is written, closed and ready
+    with open(version_module.__file__) as fid:
+        fid.read()
+    return version_str
+
+def write_vers(vers_file='wetb/__init__.py'):
+    version = get_tag(os.getcwd())
+    print('Writing version: {} in {}'.format(version, vers_file))
+    with open(vers_file, 'r') as f:
+        lines = f.readlines()
+    for n,l in enumerate(lines):
+        if l.startswith('__version__'):
+            lines[n] = "__version__ = '{}'\n".format(version[1:])
+    with open(vers_file, 'w') as f:
+        f.write(''.join(lines))
+		
+def rename_dist_file():
+    for f in os.listdir('dist'):
+        if f.endswith('whl'):
+            split = f.split('linux')
+            new_name = 'manylinux1'.join(split)
+            old_path = os.path.join('dist',f)
+            new_path = os.path.join('dist',new_name)
+            os.rename(old_path, new_path)
+    
+
+def main():
+    """Example of how to run (pytest-friendly)"""
+    if __name__ == '__main__':
+#        pass
+#        import version
+#        import app_utils
+#        git_path = os.path.dirname(app_utils.__file__) + "/../"
+#        update_git_version(version, git_path)
+#        tag = get_tag(os.getcwd())
+#        print(tag)
+#        version = get_git_version(os.getcwd())
+#        print(version)
+#        rename_dist_file()
+        write_vers()
+
+
+main()

setup.cfg

@@ -8,25 +8,25 @@ home-page = https://gitlab.windenergy.dtu.dk/toolbox/WindEnergyToolbox
 description-file = README
 # Add here all kinds of additional classifiers as defined under
 # https://pypi.python.org/pypi?%3Aaction=list_classifiers
-classifiers = Development Status :: 4 - Beta,
-              Programming Language :: Python,
-              Programming Language :: Python :: 2.7,
-              Programming Language :: Python :: 3,
-              Programming Language :: Python :: 3.3,
-              Programming Language :: Python :: 3.4,
-			  Programming Language :: Python :: 3.5,
-			  Programming Language :: Python :: 3.6,
-              Environment :: Console,
-              Intended Audience :: Education,
-              Intended Audience :: Science/Research,
-              License :: OSI Approved :: GPL License,
-              Operating System :: OS Independent,
-              Operating System :: POSIX :: Linux,
-              Operating System :: Unix,
-              Operating System :: MacOS,
-              Operating System :: Microsoft :: Windows
-              Topic :: Scientific/Engineering :: Mathematics
-
+#classifiers = Development Status :: 4 - Beta,
+#              Programming Language :: Python,
+#              Programming Language :: Python :: 2.7,
+#              Programming Language :: Python :: 3,
+#              Programming Language :: Python :: 3.3,
+#              Programming Language :: Python :: 3.4,
+#			  Programming Language :: Python :: 3.5,
+#			  Programming Language :: Python :: 3.6,
+#              Environment :: Console,
+#              Intended Audience :: Education,
+#              Intended Audience :: Science/Research,
+#              License :: OSI Approved :: GPL License,
+#              Operating System :: OS Independent,
+#              Operating System :: POSIX :: Linux,
+#              Operating System :: Unix,
+#              Operating System :: MacOS,
+#              Operating System :: Microsoft :: Windows
+#              Topic :: Scientific/Engineering :: Mathematics
+#
 [entry_points]
 # Add here console scripts like:
 # console_scripts =

setup.py

@@ -2,15 +2,13 @@
 # -*- coding: utf-8 -*-
 """
     Setup file for wetb.
-
-    This file was generated with PyScaffold 2.5, a tool that easily
-    puts up a scaffold for your new Python project. Learn more under:
-    http://pyscaffold.readthedocs.org/
 """
 
 import os
 import sys
-from setuptools import setup
+from setuptools import setup, find_packages
+import wetb
+__version__ = wetb.__version__
 
 try:
     from pypandoc import convert_file
@@ -34,11 +32,13 @@ def setup_package():
 
     needs_sphinx = {'build_sphinx', 'upload_docs'}.intersection(sys.argv)
     sphinx = ['sphinx'] if needs_sphinx else []
-    setup(setup_requires=['six', 'pyscaffold>=2.5a0,<2.6a0'] + sphinx,
+    setup(setup_requires=['six'] + sphinx,
           cmdclass = {'build_ext': build_ext},
           ext_modules = extlist,
-          use_pyscaffold=True,
-          long_description=read_md('README.md'))
+          long_description=read_md('README.md'),
+          version=__version__,
+          packages=find_packages(),
+          )
 
 
 if __name__ == "__main__":

wetb/__init__.py

@@ -5,8 +5,9 @@ from __future__ import absolute_import
 from future import standard_library
 standard_library.install_aliases()
 test = "TEST"
-try:
-    import pkg_resources
-    __version__ = pkg_resources.safe_version(pkg_resources.get_distribution(__name__).version)
-except:
-    __version__ = 'unknown'
+__version__ = '0.0.10'
+# try:
+#     import pkg_resources
+#     __version__ = pkg_resources.safe_version(pkg_resources.get_distribution(__name__).version)
+# except:
+#     __version__ = 'unknown'

wetb/standard_models/power_curve.py

+import numpy as np
+
+
+def standard_power_curve(power_norm, diameter, turbulence_intensity=.1, shear=(0, 100),
+                         rho=1.225, max_cp=.49,
+                         gear_loss_const=.01, gear_loss_var=.014, generator_loss=0.03, converter_loss=.03):
+    """Generate standard power curve
+
+    The method is extracted from an Excel spreadsheet made by Kenneth Thomsen, DTU Windenergy and
+    ported into python by Mads M. Pedersen, DTU Windenergy.
+
+    Parameters
+    ----------
+    power_norm : int or float
+        Nominal power [kW]
+    diameter : int or float
+        Rotor diameter [m]
+    turbulence_intensity : float
+        Turbulence intensity [%]
+    shear : (power shear coefficient, hub height)
+        Power shear arguments\n
+        - Power shear coeficient, alpha\n
+        - Hub height [m]
+    rho : float optional
+        Density of air [kg/m^3], defualt is 1.225
+    max_cp : float
+        Maximum power coefficient
+    gear_loss_const : float
+        Constant gear loss [%]
+    gear_loss_var : float
+        Variable gear loss [%]
+    generator_loss : float
+        Generator loss [%]
+    converter_loss : float
+
+    Examples
+    --------
+    wsp, power = standard_power_curve(10000, 178.3)
+    plot(wsp, power)
+    show()
+    """
+
+
+    area = (diameter / 2) ** 2 * np.pi
+    wsp_lst = np.arange(0.5, 25, .5)
+    sigma_lst = wsp_lst * turbulence_intensity
+
+    def norm_dist(x, my, sigma):
+        if turbulence_intensity > 0:
+            return 1 / np.sqrt(2 * np.pi * sigma ** 2) * np.exp(-(x - my) ** 2 / (2 * sigma ** 2))
+        else:
+            return x == my
+    p_aero = .5 * rho * area * wsp_lst ** 3 * max_cp / 1000
+
+    # calc power - gear, generator and conv loss
+#     gear_loss = gear_loss_const * power_norm + gear_loss_var * p_aero
+#     p_gear = p_aero - gear_loss
+#     p_gear[p_gear < 0] = 0
+#     gen_loss = generator_loss * p_gear
+#     p_gen = p_gear - gen_loss
+#     converter_loss = converter_loss * p_gen
+#     p_raw = p_gen - converter_loss
+#     p_raw[p_raw > power_norm] = power_norm
+    p_raw = p_aero - power_loss(p_aero, power_norm, gear_loss_const, gear_loss_var, generator_loss, converter_loss)
+
+    powers = []
+    shear_weighted_wsp = []
+    alpha, hub_height = shear
+    r = diameter / 2
+    z = np.linspace(hub_height - r, hub_height + r, 100, endpoint=True)
+    shear_factors = (z / hub_height) ** alpha
+    rotor_width = 2 * np.sqrt(r ** 2 - (hub_height - z) ** 2)
+
+    for wsp, sigma in zip(wsp_lst, sigma_lst):
+        shear_weighted_wsp.append(wsp * (np.trapz(shear_factors ** 3 * rotor_width, z) / (area)) ** (1 / 3))
+        ndist = norm_dist(wsp_lst, wsp, sigma)
+        powers.append((ndist * p_raw).sum() / ndist.sum())
+
+    return wsp_lst, lambda wsp : np.interp(wsp, wsp_lst, powers)
+    return wsp_lst, np.interp(shear_weighted_wsp, wsp_lst, powers)
+
+def power_loss(power_aero, power_norm, gear_loss_const=.01, gear_loss_var=.014, generator_loss=0.03, converter_loss=.03):
+    gear_loss = gear_loss_const * power_norm + gear_loss_var * power_aero
+    p_gear = power_aero - gear_loss
+    p_gear[p_gear < 0] = 0
+    gen_loss = generator_loss * p_gear
+    p_gen = p_gear - gen_loss
+    converter_loss = converter_loss * p_gen
+    p_electric = p_gen - converter_loss
+    p_electric[p_electric > power_norm] = power_norm
+    return power_aero - p_electric
+    
+
+if __name__ == '__main__':
+
+
+    from matplotlib.pyplot import plot, show
+    plot(*standard_power_curve(10000, 178.3, 0., (0, 119), gear_loss_const=.0, gear_loss_var=.0, generator_loss=0.0, converter_loss=.0))
+    plot(*standard_power_curve(10000, 178.3, 0.03, (0, 119), gear_loss_const=.0, gear_loss_var=.0, generator_loss=0.0, converter_loss=.0))
+
+
+
+
+    show()

wetb/utils/cluster_tools/pbsfile.py

@@ -142,7 +142,7 @@ class PBSMultiRunner(PBSFile):
         # find available nodes
         with open(os.environ['PBS_NODEFILE']) as fid:
             files = set([f.strip() for f in fid.readlines() if f.strip() != ''])
-        pbs_files = glob.glob('./**/*.in', recursive=True)
+        pbs_files = [os.path.join(root, f) for root, folders, f_lst in os.walk('.') for f in f_lst if f.endswith('.in')]
 
         # Make a list of [(pbs_in_filename, stdout_filename, walltime),...]
         pat = re.compile(r'[\s\S]*#\s*PBS\s+-o\s+(.*)[\s\S]*(\d\d:\d\d:\d\d)[\s\S]*')
@@ -158,7 +158,7 @@ class PBSMultiRunner(PBSFile):
         # sort wrt walltime
         pbs_info_lst = sorted(pbs_info_lst, key=lambda fow: tuple(map(int, fow[2].split(':'))))[::-1]
         # make dict {node1: pbs_info_lst1, ...} and save
-        d = {f: pbs_info_lst[i::len(files)] for i, f in enumerate(files)}
+        d = dict([(f, pbs_info_lst[i::len(files)]) for i, f in enumerate(files)])
         with open('pbs.dict', 'w') as fid:
             fid.write(str(d))
 

wetb/utils/cluster_tools/ssh_client.py

@@ -417,11 +417,10 @@ class SharedSSHClient(SSHClient):
 
 
 if __name__ == "__main__":
-    from mmpe.ui.qt_ui import QtInputUI
-    q = QtInputUI(None)
-    x = None
-    username, password = "mmpe", x.password  # q.get_login("mmpe")
-
-    client = SSHClient(host='gorm', port=22, username=username, password=password)
-    print(client.glob("*.*", ".hawc2launcher/medium1__1__"))
-    #    ssh.upload('../News.txt', 'news.txt')
+    try:
+        import x
+        username, password = 'mmpe', x.password
+        client = SSHClient(host='jess.dtu.dk', port=22, username=username, password=password)
+        print(client.execute("echo hello $USER from $HOSTNAME")[1])
+    except ImportError:
+        x = None

wetb/utils/tests/test_pbs_file.py

@@ -94,7 +94,7 @@ import re
 # find available nodes
 with open(os.environ['PBS_NODEFILE']) as fid:
     files = set([f.strip() for f in fid.readlines() if f.strip() != ''])
-pbs_files = glob.glob('./**/*.in', recursive=True)
+pbs_files = [os.path.join(root, f) for root, folders, f_lst in os.walk('.') for f in f_lst if f.endswith('.in')]
 
 # Make a list of [(pbs_in_filename, stdout_filename, walltime),...]
 pat = re.compile(r'[\s\S]*#\s*PBS\s+-o\s+(.*)[\s\S]*(\d\d:\d\d:\d\d)[\s\S]*')
@@ -110,7 +110,7 @@ pbs_info_lst = map(get_info, pbs_files)
 # sort wrt walltime
 pbs_info_lst = sorted(pbs_info_lst, key=lambda fow: tuple(map(int, fow[2].split(':'))))[::-1]
 # make dict {node1: pbs_info_lst1, ...} and save
-d = {f: pbs_info_lst[i::len(files)] for i, f in enumerate(files)}
+d = dict([(f, pbs_info_lst[i::len(files)]) for i, f in enumerate(files)])
 with open('pbs.dict', 'w') as fid:
     fid.write(str(d))