diff --git a/.gitignore b/.gitignore
index 0395feace30cb972b1648e59acf909aa13b426e8..6d3feb6cc699a5af67da7cbd28e2fcf98554c058 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,17 +3,19 @@ dist/*
*.pyc
*.pyd
*egg-info*
-
+*.so
.eggs
doctrees
-/docs/_build
-/.project
-/.pydevproject
-/.settings/org.eclipse.core.resources.prefs
-/wetb/gtsdf/tests/tmp
-/wetb/dlc/tests/test_files/res_all
-/wetb/dlc/tests/test_files/res2_all
-/wetb/hawc2/ascii2bin/ascii2bin_dist
-/wetb/hawc2/tests/test_files/htcfiles/tmp.htc
-/wetb/hawc2/ascii2bin/tests/test_files/Hawc2ascii_bin.sel
-/wetb/hawc2/ascii2bin/tests/test_files/Hawc2ascii_bin.dat
+docs/_build
+.project
+.pydevproject
+.settings/org.eclipse.core.resources.prefs
+wetb/gtsdf/tests/tmp
+wetb/dlc/tests/test_files/res_all
+wetb/dlc/tests/test_files/res2_all
+wetb/hawc2/ascii2bin/ascii2bin_dist
+wetb/hawc2/tests/test_files/htcfiles/tmp.htc
+wetb/hawc2/ascii2bin/tests/test_files/Hawc2ascii_bin.sel
+wetb/hawc2/ascii2bin/tests/test_files/Hawc2ascii_bin.dat
+wetb/prepost/tests/data/demo_dlc/remote*
+
diff --git a/README.md b/README.md
index 638cadaf0d58f942e92134df763ad96cf358b2e6..b8f2fbbb678a74baacf870f4f16355e27f5c5cc1 100644
--- a/README.md
+++ b/README.md
@@ -11,6 +11,25 @@ in [Hawc2io](wetb/hawc2/Hawc2io.py). These different implementations will be
merged in due time.
+# How to create HAWC2 DLB's and run them on a cluster
+
+The process of how to generated, run and post-process a design load basis (DLB)
+of HAWC2 simulations on a DTU Wind Energy cluster is outlined in more detail
+in the documentation:
+
+* [Auto-generation of Design Load Cases](docs/howto-make-dlcs.md)
+* [House rules mimer/hawc2sim and HAWC2 folder structure](docs/houserules-mimerhawc2sim.md)
+* [Generate DLB spreadsheets](docs/generate-spreadsheet.md)
+* [How to use the Statistics DataFrame](docs/using-statistics-df.md)
+
+You can also use the Pdap for post-processing, which includes a MS Word report
+generator based on a full DLB, a GUI for easy plotting of HAWC2 result files,
+and a Python scripting interface:
+
+* [Pdap](http://www.hawc2.dk/Download/Post-processing-tools/Pdap)
+* [Pdap report/docs](http://orbit.dtu.dk/en/publications/post-processing-of-design-load-cases-using-pdap%28827c432b-cf7d-44eb-899b-93e9c0648ca5%29.html)
+
+
# Works with Python 2 and Python 3
This module is tested for Python 2 and 3 compatibility, and works on both
@@ -40,7 +59,9 @@ in Python 2.7 by default. You can also write code that is compatible with both
* [pandas](http://pandas.pydata.org/)
-* xlrd
+* xlrd and xlwt from [python-excel](http://www.python-excel.org/)
+
+* [openpyxl](http://openpyxl.readthedocs.org/en/default/)
* h5py
@@ -61,12 +82,19 @@ Detailed installation instructions, including how to install Python from scratch
are described in the [detailed installation manual](docs/install-manual-detailed.md).
-If you know what you are doing, you can install as follows:
+If you know what you are doing, you can install as a package as follows:
```
-python setup.py
+python setup.py install
```
+Or in development mode, install from your working directory
+
+```
+pip install -e ./
+```
+
+
Or create a binary wheel distribution package with:
```
diff --git a/docs/configure-wine.md b/docs/configure-wine.md
new file mode 100644
index 0000000000000000000000000000000000000000..b3d2225cbc3bace5306bd0241259a740eea405ef
--- /dev/null
+++ b/docs/configure-wine.md
@@ -0,0 +1,54 @@
+
+Configure Wine for HAWC2
+------------------------
+
+> Note that the steps described here are executed automatically by the
+configuration script [```config-wine-hawc2.sh```]
+(https://gitlab.windenergy.dtu.dk/toolbox/pbsutils/blob/master/config-wine-hawc2.sh)
+in ```pbsutils```.
+
+
+You will also need to configure wine and place the HAWC2 executables in a
+directory that wine knows about. First, activate the correct wine environment by
+typing in a shell in the Gorm's home directory (it can be activated with
+ssh (Linux, Mac) or putty (MS Windows)):
+
+```
+g-000 $ WINEARCH=win32 WINEPREFIX=~/.wine32 wine test.exe
+```
+
+Optionally, you can also make an alias (a short format for a longer, more complex
+command). In the ```.bashrc``` file in your home directory
+(```/home/$USER/.bash_profile```), add at the bottom of the file:
+
+```
+alias wine32='WINEARCH=win32 WINEPREFIX=~/.wine32 wine'
+```
+
+Add a folder called ```bin``` to your wine system's PATH so we can copy all
+the HAWC2 executables in here:
+
+```
+WINEDIRNAME=".wine32"
+printf 'REGEDIT4\n[HKEY_CURRENT_USER\\Environment]\n"PATH"="c:\\bin"' >> ./tmp.reg
+WINEARCH=win32 WINEPREFIX=~/$WINEDIRNAME wine regedit ./tmp.reg
+rm ./tmp.reg
+```
+
+And now copy all the HAWC2 executables, DLL's (including the license manager)
+to your wine directory. You can copy all the required executables, dll's and
+the license manager are located at ```/home/MET/hawc2exe```. The following
+command will do this copying:
+
+```
+g-000 $ cp /home/MET/hawc2exe/* /home/$USER/.wine32/drive_c/bin/
+```
+
+Notice that the HAWC2 executable names are ```hawc2-latest.exe```,
+```hawc2-118.exe```, etc. By default the latest version will be used and the user
+does not need to specify this. However, when you need to compare different version
+you can easily do so by specifying which case should be run with which
+executable. The file ```hawc2-latest.exe``` will always be the latest HAWC2
+version at ```/home/MET/hawc2exe/```. When a new HAWC2 is released you can
+simply copy all the files from there again to update.
+
diff --git a/docs/generate-spreadsheet.md b/docs/generate-spreadsheet.md
new file mode 100644
index 0000000000000000000000000000000000000000..44caf721075ba25f894e708d386557a33e3e7210
--- /dev/null
+++ b/docs/generate-spreadsheet.md
@@ -0,0 +1,33 @@
+Auto-generation of DLB Spreadsheets
+===================================
+
+Introduction
+------------
+
+This manual explains how to automatically generate the set of spreadsheets that
+defines a DLB and is required as input to the pre-processor.
+
+This tool comes handy in the following scenarios:
+* a DLB for a new turbine needs to be generated;
+* a different wind turbine class needs to be evaluated;
+* a new parameter needs to be included in the htc file;
+* different parameters variations are required, e.g. different wind speed range or different number of turbulent seed.
+
+The generator of the cases uses an input spreadsheet where the cases are defined
+in a more compact way.
+The tool is based on the "tags" concept that is used for the genetaion of the htc files.
+
+Main spreatsheet
+----------------
+
+A main spreadsheet is used to defines all the DLC of the DLB. The file specifies the tags that are then required in the htc files.
+
+The file has:
+* a Main sheet where some wind turbines parameters are defined, the tags are initialized, and the definitions of turbulence and gusts are given.
+* a series of other sheets, each defining a DLC. In these sheets the tags that changes in that DLC are defined.
+
+The tags are devided into three possible different categories:
+* Constants (C). Constants are tags that do not change in a DLC, e.g. simulation time, output format, ...;
+* Variables (V). Variables are tags that define the number of cases in a DLC through their combinations, e.g. wind speed, number of turbilence seeds, wind direction, ..;
+* Functions (F). Functions are tags that depend on other tags through an expression, e.g. turbulence intensity, case name, ....
+
diff --git a/docs/houserules-mimerhawc2sim.md b/docs/houserules-mimerhawc2sim.md
new file mode 100644
index 0000000000000000000000000000000000000000..6b5e2e91932f9a1eb097c122bbd0d3abb9725d48
--- /dev/null
+++ b/docs/houserules-mimerhawc2sim.md
@@ -0,0 +1,135 @@
+
+House Rules for ```mimer/hawc2sim``` and HAWC2 model folder structure
+=====================================================================
+
+
+Objectives
+----------
+
+* Re-use turbulence boxes (save disk space)
+* Find each others simulations, review, re-run
+* Find working examples of simulations, DLB's
+* Avoid running the same DLB, simulations more than once
+
+* Disk usage quota review: administrators will create an overview of disk usage
+as used per turbine and user.
+
+
+Basic structure
+---------------
+
+The HAWC2 simulations are located on the data capacitor [mimer]
+(http://mimer.risoe.dk/mimerwiki), on the following address:
+
+```
+# on Windows, use the following address when mapping a new network drive
+\\mimer\hawc2sim
+
+# on Linux you can use sshfs or mount -t cifs
+//mimer.risoe.dk/hawc2sim
+```
+
+The following structure is currently used for this ```hawc2sim``` directory:
+* turbine model (e.g. DTU10MW, NREL5MW, etc)
+ * set ID: 2 alphabetic characters followed by 4 numbers (e.g. AA0001)
+* letters are task/project oriented, numbers are case oriented
+
+For example:
+* DTU10MW
+ * AA0001
+ * AA0002
+ * AB0001
+ * log_AA.xlsx
+ * log_BB.xlsx
+ * log_overview.xlsx
+* NREL5MW
+ * AA0001
+ * AA0002
+ * BA0001
+ * log_AA.xlsx
+ * log_BB.xlsx
+ * log_overview.xlsx
+
+
+House rules
+-----------
+
+* New Turbine model folder when a new size of the turbulence box is required
+(i.e. when the rotor size is different)
+* One "set ID" refers to one analysis, and it might contain more than one DLB
+ * If you realize more cases have to be included, add them in the same
+ "set ID". Don't start new "set ID" numbers.
+ * Each "set ID" number consists of 2 alphabetic followed by 4
+ numerical characters.
+* Log file
+ * Indicate which DLB used for the given "set ID" in the log file
+ * Indicate the changes wrt to a previous "set ID"
+ * Write clear and concise log messages so others can understand what
+ analysis or which DLB is considered
+ * Indicate in the log if something works or not.
+ * Indicate if a certain "set ID" is used for a certain publication or report
+* Keep a log file of the different letters. For instance AA might refer to load
+simulations carried out within a certain project
+* When results are outdated or wrong, delete the log and result files, but keep
+the htc, data and pbs input files so the "set ID" could be re-run again in the
+future. This is especially important if the given "set ID" has been used in a
+publication, report or Master/PhD thesis.
+
+
+File permissions
+----------------
+
+* By default only the person who generated the simulations within a given
+"set ID" can delete or modify the input files, other users have only read access.
+If you want to give everyone read and write access, you do:
+
+```
+# replace demo/AA0001 with the relevant turbine/set id
+g-000 $ cd /mnt/mimer/hawc2sim/demo
+g-000 $ chmod 777 -R AA0001
+```
+
+
+HAWC2 folder structure
+----------------------
+
+The current DLB setup assumes the following HAWC2 model folder structure:
+
+```
+|-- control
+| |-- ...
+|-- data
+| |-- ...
+|-- htc
+| |-- DLCs
+| | |-- dlc12_iec61400-1ed3.xlsx
+| | |-- dlc13_iec61400-1ed3.xlsx
+| | |-- ...
+| |-- _master
+| | `-- dtu10mw_master_C0013.htc
+```
+
+The load case definitions should be placed in Excel spreadsheets with a
+```*.xlsx``` extension. The above example shows one possible scenario whereby
+all the load case definitions are placed in ```htc/DLCs``` (all folder names
+are case sensitive). Alternatively, one can also place the spreadsheets in
+separate sub folders, for example:
+
+```
+|-- control
+| |-- ...
+|-- data
+| |-- ...
+|-- htc
+| |-- dlc12_iec61400-1ed3
+| | |-- dlc12_iec61400-1ed3.xlsx
+| |-- dlc13_iec61400-1ed3
+| | |-- dlc13_iec61400-1ed3.xlsx
+```
+
+In order to use this auto-configuration mode, there can only be one master file
+in ```_master``` that contains ```_master_``` in its file name.
+
+For the NREL5MW and the DTU10MW HAWC2 models, you can find their respective
+master files and DLC definition spreadsheet files on ```mimer/hawc2sim```.
+
diff --git a/docs/howto-make-dlcs.md b/docs/howto-make-dlcs.md
index e31b020f97b74b16021a9c2ae97a32aeab84ff3e..2c9ee437bf6b89882885f7604b6ab40fae3e67de 100644
--- a/docs/howto-make-dlcs.md
+++ b/docs/howto-make-dlcs.md
@@ -11,10 +11,11 @@ point to the gorm/jess wiki's
explain the difference in the paths seen from a windows computer and the cluster
-->
-WARNING: these notes contain configuration settings that are specif to the
+> WARNING: these notes contain configuration settings that are specif to the
DTU Wind Energy cluster Gorm. Only follow this guide in another environment if
you know what you are doing!
+
Introduction
------------
@@ -40,7 +41,7 @@ in the Excel spreadsheets): ```[Case folder]```, ```[Case id.]```, and
```[Turb base name]```.
The system will always force the values of the tags to be lower case anyway, and
-when working on Windows, this might cause some confusing and unexpected behaviour.
+when working on Windows, this might cause some confusing and unexpected behavior.
The tags themselves can have lower and upper case characters as can be seen
in the example above.
@@ -75,7 +76,7 @@ On Linux/Mac connecting to the cluster is as simple as running the following
command in the terminal:
```
-g-000 $ ssh $USER@gorm.risoe.dk
+ssh $USER@gorm.risoe.dk
```
Use your DTU password when asked. This will give you terminal access to the
@@ -93,12 +94,13 @@ Mounting the cluster discs
--------------------------
You need to be connected to the DTU network in order for this to work. You can
-also connect to the DTU network over VPN.
+also connect to the DTU network over VPN. When doing the HAWC2 simulations, you
+will interact regularly with the cluster file system and discs.
+
+### Windows
-When doing the HAWC2 simulations, you will interact regularly with the cluster
-file system and discs. It is convenient to map these discs as network
-drives (in Windows terms). Map the following network drives (replace ```$USER```
-with your user name):
+It is convenient to map these discs as network drives (in Windows terms).
+Map the following network drives (replace ```$USER``` with your user name):
```
\\mimer\hawc2sim
@@ -110,13 +112,15 @@ with the cluster discs.
Note that by default Windows Explorer will hide some of the files you will need edit.
In order to show all files on your Gorm home drive, you need to un-hide system files:
-Explorer > Organize > Folder and search options > select tab "view" > select the option to show hidden files and folders.
+Explorer > Organize > Folder and search options > select tab "view" > select the
+option to show hidden files and folders.
+
+### Unix
From Linux/Mac, you should be able to mount using either of the following
addresses:
```
//mimer.risoe.dk/hawc2sim
-//mimer.risoe.dk/well/hawc2sim
//gorm.risoe.dk/$USER
```
You can use either ```sshfs``` or ```mount -t cifs``` to mount the discs.
@@ -131,61 +135,49 @@ by editing the file ```.bash_profile``` file in your Gorm’s home directory
or create a new file with this file name in case it doesn't exist):
```
-export PATH=$PATH:/home/MET/STABCON/repositories/toolbox/pbsutils/
+export PATH=$PATH:/home/MET/repositories/toolbox/pbsutils/
```
(The corresponding open repository is on the DTU Wind Energy Gitlab server:
-[pbsutils](https://gitlab.windenergy.dtu.dk/toolbox/WindEnergyToolbox). Please
+[pbsutils](https://gitlab.windenergy.dtu.dk/toolbox/pbsutils). Please
considering reporting bugs and/or suggest improvements there. You're contributions
are much appreciated!)
-If you have been using an old version of this how-to, you might be pointing
-to an earlier version of these tools/utils and its reference should be removed
-from your ```.bash_profile``` file:
-
-```
-export PATH=$PATH:/home/MET/STABCON/repositories/cluster-tools/
-```
+> If you have been using an old version of this how-to, you might be pointing
+to an earlier version of these tools/utils and any references containing
+```cluster-tools``` or ```prepost``` should be removed
+from your ```.bash_profile``` and/or ```.bashrc``` file on your gorm home drive.
-After modifying ```.bash_profile```, save and close it. Then, in the terminal, run the command:
+After modifying ```.bash_profile```, save and close it. Then, in the terminal,
+run the command (or logout and in again to be safe):
```
g-000 $ source ~/.bash_profile
```
-In order for any changes made in ```.bash_profile``` to take effect, you need to either ```source``` it (as shown above), or log out and in again.
-You will also need to configure wine and place the HAWC2 executables in a
-directory that wine knows about. First, activate the correct wine environment by
-typing in a shell in the Gorm's home directory (it can be activated with
-ssh (Linux, Mac) or putty (MS Windows)):
+You will also need to configure wine and place the HAWC2 executables in your
+local wine directory, which by default is assumed to be ```~/.wine32```, and
+```pbsutils``` contains and automatic configuration script you can run:
```
-g-000 $ WINEARCH=win32 WINEPREFIX=~/.wine32 wine test.exe
+g-000 $ config-wine-hawc2.sh
```
-Optionally, you can also make an alias (a short format for a longer, more complex
-command). In the ```.bashrc``` file in your home directory
-(```/home/$USER/.bash_profile```), add at the bottom of the file:
+If you need more information on what is going on, you can read a more detailed
+description [here]
+(https://gitlab.windenergy.dtu.dk/toolbox/WindEnergyToolbox/blob/master/docs/configure-wine.md).
-```
-alias wine32='WINEARCH=win32 WINEPREFIX=~/.wine32 wine'
-```
-
-And now copy all the HAWC2 executables, DLL's (including the license manager)
-to your wine directory. You can copy all the required executables, dll's and
-the license manager are located at ```/home/MET/hawc2exe```. The following
-command will do this copying:
-
-```
-g-000 $ cp /home/MET/hawc2exe/* /home/$USER/.wine32/drive_c/windows/system32
-```
+All your HAWC2 executables and DLL's are now located
+at ```/home/$USER/.wine32/drive_c/bin```.
Notice that the HAWC2 executable names are ```hawc2-latest.exe```,
```hawc2-118.exe```, etc. By default the latest version will be used and the user
does not need to specify this. However, when you need to compare different version
you can easily do so by specifying which case should be run with which
-executable. The file ```hawc2-latest.exe``` will always be the latest HAWC2
-version at ```/home/MET/hawc2exe/```. When a new HAWC2 is released you can
-simply copy all the files from there again to update.
+executable.
+
+Alternatively you can also include all the DLL's and executables in the root of
+your HAWC2 model folder. Executables and DLL's placed in the root folder take
+precedence over the ones placed in ```/home/$USER/.wine32/drive_c/bin```.
Log out and in again from the cluster (close and restart PuTTY).
@@ -196,73 +188,70 @@ g-000 $ wine32 hawc2-latest htc/some-intput-file.htc
```
-Method A: Generating htc input files on the cluster
----------------------------------------------------
-
-Use ssh (Linux, Mac) or putty (MS Windows) to connect to the cluster.
-
-With qsub-wrap.py the user can wrap a PBS launch script around any executable or
-Python/Matlab/... script. In doing so, the executable/Python script will be
-immediately submitted to the cluster for execution. By default, the Anaconda
-Python environment in ```/home/MET/STABCON/miniconda``` will be activated. The
-Anaconda Python environment is not relevant, and can be safely ignored, if the
-executable does not have anything to do with Python.
+Updating local HAWC2 executables
+--------------------------------
-In order to see the different options of this qsub-wrap utility, do:
+When there is a new version of HAWC2, or when a new license manager is released,
+you can update your local wine directory as follows:
```
-g-000 $ qsub-wrap.py --help
+g-000 $ cp /home/MET/hawc2exe/* /home/$USER/.wine32/drive_c/bin/
```
-For example, in order to generate the default IEC DLCs:
+The file ```hawc2-latest.exe``` will always be the latest HAWC2
+version at ```/home/MET/hawc2exe/```. When a new HAWC2 is released you can
+simply copy all the files from there again to update.
+
+
+HAWC2 model folder structure and results on mimer/hawc2sim
+----------------------------------------------------------
+
+See [house rules on mimer/hawc2sim]
+(https://gitlab.windenergy.dtu.dk/toolbox/WindEnergyToolbox/blob/master/docs/houserules-mimerhawc2sim.md)
+for a more detailed description.
+
+
+Method A: Generating htc input files on the cluster (recommended)
+-----------------------------------------------------------------
+
+Use ssh (Linux, Mac) or putty (MS Windows) to connect to the cluster.
+
+In order to simplify things, we're using ```qsub-wrap.py``` from ```pbsutils```
+(which we added under the [preparation]/(#preparation) section) in order to
+generate the htc files. It will execute, on a compute node, any given Python
+script in a pre-installed Python environment that has the Wind Energy Toolbox
+installed.
+
+For the current implementation of the DLB the following template is available:
```
-g-000 $ cd path/to/HAWC2/model # folder where the hawc2 model is located
-g-000 $ qsub-wrap.py -f /home/MET/STABCON/repositories/prepost/dlctemplate.py -c python --prep
+/home/MET/repositories/toolbox/WindEnergyToolbox/wetb/prepost/dlctemplate.py
```
-Note that the following folder structure for the HAWC2 model is assumed:
+And the corresponding definitions of all the different load cases can be copied
+from here (valid for the DTU10MW):
```
-|-- control
-| |-- ...
-|-- data
-| |-- ...
-|-- htc
-| |-- DLCs
-| | |-- dlc12_iec61400-1ed3.xlsx
-| | |-- dlc13_iec61400-1ed3.xlsx
-| | |-- ...
-| |-- _master
-| | `-- dtu10mw_master_C0013.htc
+/mnt/mimer/hawc2sim/DTU10MW/C0020/htc/DLCs
```
-The load case definitions should be placed in Excel spreadsheets with a
-```*.xlsx``` extension. The above example shows one possible scenario whereby
-all the load case definitions are placed in ```htc/DLCs``` (all folder names
-are case sensitive). Alternatively, one can also place the spreadsheets in
-separate sub folders, for example:
+For example, in order to generate all the HAWC2 htc input files and the
+corresponding ```*.p``` cluster launch files using this default DLB setup with:
```
-|-- control
-| |-- ...
-|-- data
-| |-- ...
-|-- htc
-| |-- dlc12_iec61400-1ed3
-| | |-- dlc12_iec61400-1ed3.xlsx
-| |-- dlc13_iec61400-1ed3
-| | |-- dlc13_iec61400-1ed3.xlsx
+g-000 $ cd /mnt/mimer/hawc2sim/demo/A0001 # folder where the hawc2 model is located
+g-000 $ qsub-wrap.py -f /home/MET/repositories/toolbox/WindEnergyToolbox/wetb/prepost/dlctemplate.py --prep
```
-In order to use this auto-configuration mode, there can only be one master file
-in ```_master``` that contains ```_master_``` in its file name.
+You could consider adding ```dlctemplate.py``` into the turbine folder or in
+the simulation set id folder for your convenience:
-For the NREL5MW and the DTU10MW HAWC2 models, you can find their respective
-master files and DLC definition spreadsheet files on Mimer. When connected
-to Gorm over SSH/PuTTY, you will find these files at:
```
-/mnt/mimer/hawc2sim # (when on Gorm)
+g-000 $ cd /mnt/mimer/hawc2sim/demo/
+# copy the dlctemplate to your turbine model folder and rename to myturbine.py
+g-000 $ cp /home/MET/repositories/toolbox/WindEnergyToolbox/wetb/prepost/dlctemplate.py ./myturbine.py
+g-000 $ cd A0001
+g-000 $ qsub-wrap.py -f ../myturbine.py --prep
```
@@ -278,20 +267,17 @@ First activate the Anaconda Python environment by typing:
```bash
# add the Anaconda Python environment paths to the system PATH
-g-000 $ export PATH=/home/MET/STABCON/miniconda/bin:$PATH
+g-000 $ export PATH=/home/python/miniconda3/bin:$PATH
# activate the custom python environment:
-g-000 $ source activate anaconda
-# add the Pythone libraries to the PYTHONPATH
-g-000 $ export PYTHONPATH=/home/MET/STABCON/repositories/prepost:$PYTHONPATH
-g-000 $ export PYTHONPATH=/home/MET/STABCON/repositories/pythontoolbox/fatigue_tools:$PYTHONPATH
-g-000 $ export PYTHONPATH=/home/MET/STABCON/repositories/pythontoolbox:$PYTHONPATH
-g-000 $ export PYTHONPATH=/home/MET/STABCON/repositories/MMPE:$PYTHONPATH
+g-000 $ source activate wetb_py3
```
For example, launch the auto-generation of DLCs input files:
```
-g-000 $ cd path/to/HAWC2/model # folder where the hawc2 model is located
-g-000 $ python /home/MET/STABCON/repositories/prepost/dlctemplate.py --prep
+# folder where the HAWC2 model is located
+g-000 $ cd /mnt/mimer/hawc2sim/demo/AA0001
+# assuming myturbine.py is copy of dlctemplate.py and is placed one level up
+g-000 $ python ../myturbine.py --prep
```
Or start an interactive IPython shell:
@@ -309,16 +295,18 @@ jammed.
Method C: Generating htc input files locally
--------------------------------------------
-This approach gives you total freedom, but is also more difficult since you
-will have to have fully configured Python environment installed locally.
+This approach gives you more flexibility and room for custimizations, but you
+will need to install a Python environment with all its dependencies locally.
Additionally, you need access to the cluster discs from your local workstation.
-Method C is not documented yet.
+
+The installation procedure for wetb is outlined in the [installation manual]
+(https://gitlab.windenergy.dtu.dk/toolbox/WindEnergyToolbox/blob/master/docs/install-manual-detailed.md).
Optional configuration
----------------------
-Optional tags that can be set in the Excel spreadsheet, and their corresponding
+Optional tags that can be set in the Excel spreadsheet and their corresponding
default values are given below. Beside a replacement value in the master htc
file, there are also special actions connected to these values. Consequently,
these tags have to be present. When removed, the system will stop working properly.
@@ -362,8 +350,23 @@ This zip file will be extracted into the execution directory (```[run_dir]```).
After the model has ran on the node, only the files that have been created
during simulation time in the ```[log_dir]```, ```[res_dir]```,
```[animation_dir]```, and ```[eigenfreq_dir]``` will be copied back.
-Optionally, on can also copy back the turbulence files, and other explicitly
-defined files [TODO: expand manual here].
+
+
+### Advanced configuration options
+
+> Note that not all features are documented yet...
+
+Special tags: copy special result files from the compute node back to the HAWC2
+working directory on the network drive, and optionally rename the file in case
+it would otherwise be overwritten by other cases in your DLB:
+* ```[copyback_files] = ['ESYSMooring_init.dat']```
+* ```[copyback_frename] = ['path/to/ESYSMooring_init_vXYZ.dat']```, optionally specify
+a different file path/name
+
+Copy files from the HAWC2 working directory with a special name to the compute
+node for which the a fixed file name is assumed
+* ```[copyto_files] = ['path/to/ESYSMooring_init_vXYZ.dat']```
+* ```[copyto_generic] = ['ESYSMooring_init.dat']```
Launching the jobs on the cluster
@@ -429,12 +432,20 @@ options:
```
Then launch the actual jobs (each job is a ```*.p``` file in ```pbs_in```) using
-100 cpu's, and using a compute node instead of the login node (see you can exit
-the ssh/putty session without interrupting the launching process):
+100 cpu's:
```bash
-g-000 $ cd path/to/HAWC2/model
-g-000 $ launch.py -n 100 --node
+g-000 $ cd /mnt/mimer/hawc2sim/demo/A0001
+g-000 $ launch.py -n 100 -p pbs_in/
+```
+
+If the launching process requires hours, and you have to close you SHH/PuTTY
+session before it reaches the end, you should use the ```--node``` argument so
+the launching process will take place on a dedicated node:
+
+```bash
+g-000 $ cd /mnt/mimer/hawc2sim/demo/A0001
+g-000 $ launch.py -n 100 -p pbs_in/ --node
```
@@ -520,10 +531,11 @@ htc files, but now we set different flags. For example, for checking the log
files, calculating the statistics, the AEP and the life time equivalent loads:
```
-g-000 $ qsub-wrap.py -f /home/MET/STABCON/repositories/prepost/dlctemplate.py -c python --years=25 --neq=1e7 --stats --check_logs --fatigue
+# myturbine.py (copy of dlctemplate.py) is assumed to be located one folder up
+g-000 $ qsub-wrap.py -f ../myturbine.py --years=25 --neq=1e7 --stats --check_logs --fatigue
```
-Other options for the ```dlctemplate.py``` script:
+Other options for the original ```dlctemplate.py``` script:
```
usage: dlctemplate.py [-h] [--prep] [--check_logs] [--stats] [--fatigue]
@@ -536,32 +548,34 @@ optional arguments:
-h, --help show this help message and exit
--prep create htc, pbs, files (default=False)
--check_logs check the log files (default=False)
- --stats calculate statistics (default=False)
+ --stats calculate statistics and 1Hz equivalent loads (default=False)
--fatigue calculate Leq for a full DLC (default=False)
--csv Save data also as csv file (default=False)
--years YEARS Total life time in years (default=20)
--no_bins NO_BINS Number of bins for fatigue loads (default=46)
- --neq NEQ Equivalent cycles neq (default=1e6)
+ --neq NEQ Equivalent cycles neq, default 1 Hz equivalent load
+ (neq = simulation duration in seconds)
--envelopeblade calculate the load envelope for sensors on the blades
--envelopeturbine calculate the load envelope for sensors on the turbine
```
-The load envelopes are computed for sensors specified in the
-```dlctemplate.py``` file. The sensors are specified in a list of lists. The
+The load envelopes are computed for sensors specified in the
+```myturbine.py``` file. The sensors are specified in a list of lists. The
inner list contains the sensors at one location. The envelope is computed for
the first two sensors of the inner list and the other sensors are used to
-retrieve the remaining loads defining the load state occurring at the same
+retrieve the remaining loads defining the load state occurring at the same
instant. The outer list is used to specify sensors at different locations.
The default values for the blade envelopes are used to compute the Mx-My
-envelopes and retrieve the Mz-Fx-Fy-Fz loads occuring at the same moment.
+envelopes and retrieve the Mz-Fx-Fy-Fz loads occurring at the same moment.
+
Debugging
---------
Any output (everything that involves print statements) generated during the
-post-processing of the simulations using ```dlctemplate.py``` is captured in
-the ```pbs_out/qsub-wrap_dlctemplate.py.out``` file, while exceptions and errors
-are redirected to the ```pbs_out/qsub-wrap_dlctemplate.py.err``` text file.
+post-processing of the simulations using ```myturbine.py``` is captured in
+the ```pbs_out/qsub-wrap_myturbine.py.out``` file, while exceptions and errors
+are redirected to the ```pbs_out/qsub-wrap_myturbine.py.err``` text file.
The output and errors of HAWC2 simulations can also be found in the ```pbs_out```
directory. The ```.err``` and ```.out``` files will be named exactly the same
diff --git a/docs/install-anaconda.md b/docs/install-anaconda.md
new file mode 100644
index 0000000000000000000000000000000000000000..db7b2bd7d12655f48e5eb4ccb9f1fe2b5371e4b5
--- /dev/null
+++ b/docs/install-anaconda.md
@@ -0,0 +1,15 @@
+
+# Installation manual
+
+
+## Anaconda or Miniconda
+
+```
+conda update --all
+conda create -n wetb_py3 python=3.5
+source activate wetb_py3
+conda install setuptools_scm future h5py pytables pytest nose sphinx
+conda install scipy pandas matplotlib cython xlrd coverage xlwt
+pip install pyscaffold pytest-cov
+```
+
diff --git a/docs/install-manual-detailed.md b/docs/install-manual-detailed.md
index 150e9266a42c4195e7797d3b6a91a194219b8b56..aa2216a782555a026b521dc1e46eb6f0a60f6d8b 100644
--- a/docs/install-manual-detailed.md
+++ b/docs/install-manual-detailed.md
@@ -1,5 +1,5 @@
-!! This guide is not finished, and might contain innacuracies. Please report
+> !! This guide is not finished, and might contain innacuracies. Please report
any mistakes/bugs by creating an
[issue](https://gitlab.windenergy.dtu.dk/toolbox/WindEnergyToolbox/issues).
This is a WIP (Work In Progress) !!
@@ -29,7 +29,7 @@ python-pytables.
* Other tools you will need:
-> git gcc
+> git gcc gcc-fortran (gfortran)
## Mac
@@ -165,14 +165,15 @@ activate py27
* Install the necessary Python dependencies using the conda package manager:
```
-conda install scipy pandas matplotlib cython xlrd pytables sphinx nose setuptools_scm future h5py
+conda install setuptools_scm future h5py pytables pytest pytest-cov nose sphinx
+conda install scipy pandas matplotlib cython xlrd sphinx
```
* Not all packages are available in the conda repositories, but they can be
easily installed with pip:
```
-pip install pyscaffold pytest pytest-cov
+pip install pyscaffold pytest
```
diff --git a/requirements.txt b/requirements.txt
index 4f4ab5838748fa63eb598afca49c3f6cd8148ef6..1c12bab28021d3dd10594cea6cb59de041bd0684 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,6 +5,8 @@ scipy>=0.9
matplotlib
pytest
xlrd
+xlwt
+openpyxl
h5py
pandas
tables
diff --git a/wetb/prepost/GenerateDLCs.py b/wetb/prepost/GenerateDLCs.py
index 3bcf9bd2d5db7d36c437f551bc6f0994da094196..4563ccbe28deb9db7f929de67cc4cecdaf270b5b 100644
--- a/wetb/prepost/GenerateDLCs.py
+++ b/wetb/prepost/GenerateDLCs.py
@@ -14,6 +14,8 @@ from __future__ import absolute_import
from numpy import floor, arctan, pi
import pandas as pd
import xlrd
+from argparse import ArgumentParser
+import os
def multi_for(iterables):
@@ -230,7 +232,9 @@ class GenerateDLCCases(GeneralDLC):
self.add_formulas(dlc, general_functions)
self.eval_formulas(dlc)
df = pd.DataFrame(dlc)
- df.to_excel(folder+sheet.name+'.xls', index=False)
+ if not os.path.exists(folder):
+ os.makedirs(folder)
+ df.to_excel(os.path.join(folder, sheet.name+'.xlsx'), index=False)
class RunTest():
@@ -257,6 +261,12 @@ class RunTest():
assert_frame_equal(book1, book2, check_dtype=False)
if __name__ == '__main__':
+
+ parser = ArgumentParser(description = "generator of DLB spreadsheets")
+ parser.add_argument('--master', type=str, default='DLCs.xlsx', action='store',
+ dest='filename', help='Master spreadsheet file')
+ parser.add_argument('--folder', type=str, default='', action='store',
+ dest='folder', help='Destination folder name')
+ opt = parser.parse_args()
DLB = GenerateDLCCases()
- DLB.execute()
- pass
+ DLB.execute(filename=opt.filename, folder=opt.folder)
\ No newline at end of file
diff --git a/wetb/prepost/Simulations.py b/wetb/prepost/Simulations.py
index a8abd120fc7b0bebedb5994d95d327bf82e6833b..fa55cdccdbcab0915d37d27ba0fb7f414d63e4f2 100755
--- a/wetb/prepost/Simulations.py
+++ b/wetb/prepost/Simulations.py
@@ -1161,6 +1161,20 @@ def post_launch(cases, save_iter=False):
return cases_fail
+def copy_pbs_in_failedcases(cases_fail, pbs_fail='pbs_in_fail'):
+ """
+ Copy all the pbs_in files from failed cases to a new directory so it
+ is easy to re-launch them
+ """
+
+ for cname in cases_fail.keys():
+ case = cases_fail[cname]
+ pbs_in_fname = '%s.p' % (case['[case_id]'])
+ pbs_in_dir = case['[pbs_in_dir]'].replace('pbs_in', pbs_fail)
+ run_dir = case['[run_dir]']
+ fname = os.path.join(run_dir, pbs_in_dir, pbs_in_fname)
+
+
def logcheck_case(errorlogs, cases, case, silent=False):
"""
Check logfile of a single case
@@ -1721,9 +1735,9 @@ class HtcMaster(object):
# load the file:
if not self.silent:
print('loading master: ' + fpath)
- FILE = open(fpath, 'r')
- lines = FILE.readlines()
- FILE.close()
+
+ with open(fpath, 'r') as f:
+ lines = f.readlines()
# regex for finding all tags in a line
regex = re.compile('(\\[.*?\\])')
@@ -1919,7 +1933,6 @@ class PBS(object):
# in case you want to redirect stdout to /dev/nul
# self.wine_appendix = '> /dev/null 2>&1'
self.wine_appendix = ''
- self.wine_dir = '/home/dave/.wine32/drive_c/bin'
# /dev/shm should be the RAM of the cluster
# self.node_run_root = '/dev/shm'
self.node_run_root = '/scratch'
@@ -2280,11 +2293,6 @@ class PBS(object):
self.pbs += 'echo ""\n'
self.pbs += 'echo "Execute commands on scratch nodes"\n'
self.pbs += 'cd %s/$USER/$PBS_JOBID\n' % self.node_run_root
-# # also copy all the HAWC2 exe's to the scratch dir
-# self.pbs += "cp -R %s/* ./\n" % self.wine_dir
-# # custom name hawc2 exe
-# self.h2_new = tag_dict['[hawc2_exe]'] + '-' + jobid + '.exe'
-# self.pbs += "mv %s.exe %s\n" % (tag_dict['[hawc2_exe]'], self.h2_new)
def ending(self, pbs_path):
"""
@@ -2669,7 +2677,7 @@ class ErrorLogs(object):
if self.cases is not None:
case = self.cases[fname.replace('.log', '.htc')]
dt = float(case['[dt_sim]'])
- time_steps = float(case['[time_stop]']) / dt
+ time_steps = int(float(case['[time_stop]']) / dt)
iterations = np.ndarray( (time_steps+1,3), dtype=np.float32 )
else:
iterations = np.ndarray( (len(lines),3), dtype=np.float32 )
@@ -4489,7 +4497,7 @@ class Cases(object):
# we assume the run_dir (root) is the same every where
run_dir = self.cases[case]['[run_dir]']
- fname = os.path.join(run_dir, 'htc', 'DLCs', 'dlc_config.xlsx')
+ fname = os.path.join(run_dir, 'dlc_config.xlsx')
dlc_cfg = dlc.DLCHighLevel(fname, shape_k=wb.shape_k)
# if you need all DLCs, make sure to have %s in the file name
dlc_cfg.res_folder = os.path.join(run_dir, res_dir, dlc_folder)
@@ -4629,7 +4637,7 @@ class Cases(object):
# we assume the run_dir (root) is the same every where
run_dir = self.cases[list(self.cases.keys())[0]]['[run_dir]']
- fname = os.path.join(run_dir, 'htc', 'DLCs', 'dlc_config.xlsx')
+ fname = os.path.join(run_dir, 'dlc_config.xlsx')
dlc_cfg = dlc.DLCHighLevel(fname, shape_k=wb.shape_k)
# if you need all DLCs, make sure to have %s in the file name
dlc_cfg.res_folder = os.path.join(run_dir, res_dir, dlc_folder)
diff --git a/wetb/prepost/dlcdefs.py b/wetb/prepost/dlcdefs.py
index 747e775320bb335f4f80eaed9063400ff4c4ca81..78a968547c910f712d245ef27a1bfc30bc4211ec 100644
--- a/wetb/prepost/dlcdefs.py
+++ b/wetb/prepost/dlcdefs.py
@@ -46,7 +46,7 @@ def configure_dirs(verbose=False):
raise ValueError('Could not find master file in htc/_master')
MASTERFILE = master
P_MASTERFILE = os.path.join(P_SOURCE, 'htc%s_master%s' % (os.sep, os.sep))
- POST_DIR = os.path.join(p_run_root, PROJECT, 'prepost-data%s' % os.sep)
+ POST_DIR = os.path.join(p_run_root, PROJECT, sim_id, 'prepost-data%s' % os.sep)
if verbose:
print('='*79)
diff --git a/wetb/prepost/dlctemplate.py b/wetb/prepost/dlctemplate.py
index b6240f46840503754662d6b80b23c03bee436105..bf42c8a2e9d0d8e95b650b584d649dbbc2157c3c 100755
--- a/wetb/prepost/dlctemplate.py
+++ b/wetb/prepost/dlctemplate.py
@@ -14,8 +14,6 @@ from builtins import range
from future import standard_library
standard_library.install_aliases()
-
-
import os
import socket
from argparse import ArgumentParser
@@ -267,11 +265,14 @@ def launch_param(sim_id):
def post_launch(sim_id, statistics=True, rem_failed=True, check_logs=True,
force_dir=False, update=False, saveinterval=2000, csv=False,
- m=[1, 3, 4, 5, 6, 8, 10, 12, 14], neq=1e6, no_bins=46,
+ m=[1, 3, 4, 5, 6, 8, 10, 12, 14], neq=None, no_bins=46,
years=20.0, fatigue=True, nn_twb=1, nn_twt=20, nn_blr=4, A=None,
save_new_sigs=False, envelopeturbine=False, envelopeblade=False,
save_iter=False, AEP=False):
+ if neq < 0:
+ neq = None
+
# =========================================================================
# check logfiles, results files, pbs output files
# logfile analysis is written to a csv file in logfiles directory
@@ -395,7 +396,8 @@ if __name__ == '__main__':
parser.add_argument('--check_logs', action='store_true', default=False,
dest='check_logs', help='check the log files')
parser.add_argument('--stats', action='store_true', default=False,
- dest='stats', help='calculate statistics')
+ dest='stats', help='calculate statistics and 1Hz '
+ 'equivalent loads')
parser.add_argument('--fatigue', action='store_true', default=False,
dest='fatigue', help='calculate Leq for a full DLC')
parser.add_argument('--AEP', action='store_true', default=False,
@@ -407,8 +409,10 @@ if __name__ == '__main__':
dest='years', help='Total life time in years')
parser.add_argument('--no_bins', type=float, default=46.0, action='store',
dest='no_bins', help='Number of bins for fatigue loads')
- parser.add_argument('--neq', type=float, default=1e6, action='store',
- dest='neq', help='Equivalent cycles neq')
+ parser.add_argument('--neq', type=float, default=-1.0, action='store',
+ dest='neq', help='Equivalent cycles neq, default 1 Hz '
+ 'equivalent load (neq = simulation '
+ 'duration in seconds)')
parser.add_argument('--nn_twt', type=float, default=20, action='store',
dest='nn_twt', help='Node number tower top')
parser.add_argument('--nn_blr', type=float, default=4, action='store',
@@ -425,12 +429,6 @@ if __name__ == '__main__':
dest='envelopeturbine', help='Compute envelopeturbine')
opt = parser.parse_args()
- # auto configure directories: assume you are running in the root of the
- # relevant HAWC2 model
- # and assume we are in a simulation case of a certain turbine/project
- P_RUN, P_SOURCE, PROJECT, sim_id, P_MASTERFILE, MASTERFILE, POST_DIR \
- = dlcdefs.configure_dirs(verbose=True)
-
# TODO: use arguments to determine the scenario:
# --plots, --report, --...
@@ -457,6 +455,12 @@ if __name__ == '__main__':
# saveinterval=2000, csv=True, fatigue_cycles=True, fatigue=False)
# -------------------------------------------------------------------------
+ # auto configure directories: assume you are running in the root of the
+ # relevant HAWC2 model
+ # and assume we are in a simulation case of a certain turbine/project
+ P_RUN, P_SOURCE, PROJECT, sim_id, P_MASTERFILE, MASTERFILE, POST_DIR \
+ = dlcdefs.configure_dirs(verbose=True)
+
# create HTC files and PBS launch scripts (*.p)
if opt.prep:
print('Start creating all the htc files and pbs_in files...')
@@ -473,5 +477,5 @@ if __name__ == '__main__':
envelopeblade=opt.envelopeblade)
if opt.dlcplot:
sim_ids = [sim_id]
- figdir = os.path.join(P_RUN, '..', 'figures/%s' % '-'.join(sim_ids))
+ figdir = os.path.join(POST_DIR, 'figures/%s' % '-'.join(sim_ids))
dlcplots.plot_stats2(sim_ids, [POST_DIR], fig_dir_base=figdir)