Skip to content
Snippets Groups Projects
Commit 6425c3c2 authored by David Verelst's avatar David Verelst
Browse files

prepost.GenerateDLCs: add tests, small code clean-up, rem binary results

parent 554fef86
Branches ozgo
No related tags found
1 merge request!168GenerateDLCs.py is updated for xlsx formats
Pipeline #20943 passed
Stage: test
Stage: deploy
Hide whitespace changes
Inline Side-by-side
Showing
with 159 additions and 38 deletions
.gitlab-ci.yml
+ 8
− 7
View file @ 6425c3c2
......@@ -3,7 +3,7 @@ image: dtuwindenergy/wetb
test-3.4:
stage:
stage:
test
# except:
# - test_pypi
......@@ -17,7 +17,8 @@ test-3.4:
- python3 -m pytest --cov-report term-missing:skip-covered --cov=wetb
tags:
- python
# ===== build documentation =====
pages: # "pages" is a job specifically for GitLab pages [1]
stage: # build, test, deploy defined by default [2]
......@@ -34,14 +35,14 @@ pages: # "pages" is a job specifically for GitLab pages [1]
- public
only: # only run for these branches
- master
- /^test_doc.*/
- /^test_doc.*/
tags: # only runners with this tag can do the job [3]
- python
# ===== BUILD WHEELS AND UPLOAD TO PYPI =====
pypi_linux:
stage:
stage:
deploy
only:
- tags
......@@ -55,12 +56,12 @@ pypi_linux:
- python3 -c 'from git_utils import rename_dist_file; rename_dist_file()'
- twine upload dist/* -u $TWINE_USERNAME -p $TWINE_PASSWORD
#- twine upload --repository-url https://test.pypi.org/legacy/ dist/* -u $TWINE_USERNAME -p $TWINE_PASSWORD # for testing purposes
tags:
tags:
- python
pypi_windows:
stage:
stage:
deploy
only:
- tags
......@@ -69,6 +70,6 @@ pypi_windows:
- c:/Anaconda3/envs/WindEnergyToolbox/python.exe setup.py bdist_wheel
- twine upload dist/* -u %TWINE_USERNAME% -p %TWINE_PASSWORD%
#- twine upload --repository-url https://test.pypi.org/legacy/ dist/* -u %TWINE_USERNAME% -p %TWINE_PASSWORD% # for testing purposes
tags:
tags:
- CPAV_old
requirements.txt
+ 0
− 2
View file @ 6425c3c2
......@@ -5,7 +5,6 @@ numpy>=1.4
scipy>=0.9
matplotlib
pytest
xlrd
xlwt
openpyxl
h5py
......@@ -15,7 +14,6 @@ future
paramiko
psutil
pbr
PyScaffold
pytest-cov
sshtunnel
xlsxwriter
......
setup.py
+ 0
− 1
View file @ 6425c3c2
......@@ -61,7 +61,6 @@ def setup_package(build_ext_switch=True):
'pandas',
'matplotlib',
'cython',
'xlrd',
'coverage',
'xlwt',
'openpyxl',
......
wetb/prepost/GenerateDLCs.py
+ 29
− 28
View file @ 6425c3c2
......@@ -14,9 +14,7 @@ from __future__ import absolute_import
from numpy import (floor, arctan, pi, log, log10, sin, cos, tan, e, arcsin,
arccos)
import pandas as pd
from openpyxl import Workbook
from openpyxl import load_workbook
import xlrd
from argparse import ArgumentParser
import os
......@@ -101,6 +99,8 @@ class GeneralDLC(object):
counter = floor(irow/len_wsp) + 1
value = '%4.4i' % (1000*counter + row[i_wsp] + 1)
elif variables_order[icol] == '[wave_seed]':
# FIXME: shouldn't we also have unique wave seeds??
# that is not the case with this implementation
value = '%4.4i' % (100*(row[i_wsp]+1) + row[i_wave_seed] + 1)
# value = '%4.4i' % (1000*counter + row[i_wsp] + 101)
# value = '%4.4i' % (irow+1)
......@@ -219,29 +219,32 @@ class GenerateDLCCases(GeneralDLC):
def execute(self, filename='DLCs.xlsx', folder='', isheets=None):
book = load_workbook(filename, data_only=True)
book = load_workbook(filename, data_only=True)
# Read all the initialization constants and functions of the main sheet
# The main sheet is assumed to be the first one (i=0)
general_constants = {}
general_functions = {}
sheet = book.worksheets[0]
# note that sheet.cell(i,j) is using 1-based indexing (row nr 1)
# first column is just for readability
for colnr in range(2, sheet.max_column+1):
if sheet.cell(10, colnr).value != None:
general_constants[str(sheet.cell(10, colnr).value)] = \
sheet.cell(11, colnr).value
if sheet.cell(14, colnr).value != None:
general_functions[str(sheet.cell(14, colnr).value)] = \
sheet.cell(15, colnr).value
if isheets is None:
isheets = list(range(1,len(book.sheetnames)))
# refer to sheet number, so 1-based indexing
isheets = list(range(1,len(book.sheetnames)))
# Loop through all the sheets. Each sheet correspond to a DLC.
for isheet in isheets:
# Read all the initialization constants and functions in the
# first sheet
general_constants = {}
general_functions = {}
sheet = book.worksheets[0]
for i in range(2, sheet.max_column+1):
if sheet.cell(10, i).value != None:
general_constants[str(sheet.cell(10, i).value)] = \
sheet.cell(11, i).value
if sheet.cell(14, i).value != None:
general_functions[str(sheet.cell(14, i).value)] = \
sheet.cell(15, i).value
sheet = book.worksheets[isheet]
print('Sheet #%i' % isheet, book.sheetnames[isheet])
# Read the actual sheet.
constants = {}
......@@ -265,23 +268,21 @@ class GenerateDLCCases(GeneralDLC):
formulas[tag] = str(sheet.cell(3, i).value)
dlc = {}
general_constants = self.remove_from_dict(variables,
general_constants)
general_constants = self.remove_from_dict(constants,
general_constants)
general_functions = self.remove_from_dict(formulas,
general_functions)
# make copies of the general constants and functions since remove
# will otherwise remove them for the following sheets as well
sheet_gen_con = self.remove_from_dict(variables, general_constants.copy())
sheet_gen_con = self.remove_from_dict(constants, sheet_gen_con)
sheet_gen_fun = self.remove_from_dict(formulas, general_functions.copy())
self.add_variables_tag(dlc, variables, variables_order)
self.add_constants_tag(dlc, general_constants)
self.add_constants_tag(dlc, sheet_gen_con)
self.add_constants_tag(dlc, constants)
self.add_formulas(dlc, formulas)
self.add_formulas(dlc, general_functions)
self.add_formulas(dlc, sheet_gen_fun)
# TODO: before eval, check if all tags in formula's are present
self.eval_formulas(dlc)
df = pd.DataFrame(dlc)
if not os.path.exists(folder):
os.makedirs(folder)
os.makedirs(folder, exist_ok=True)
df.to_excel(os.path.join(folder, book.sheetnames[isheet]+'.xlsx'), index=False)
......
wetb/prepost/tests/data/demo_gendlc/DLCs/DLC12.xlsx deleted 100644 → 0
+ 0
− 0
View file @ 554fef86
File deleted
wetb/prepost/tests/data/demo_gendlc/DLCs/DLC13.xlsx deleted 100644 → 0
+ 0
− 0
View file @ 554fef86
File deleted
wetb/prepost/tests/data/demo_gendlc/ref/DLC12.csv 0 → 100644
+ 37
− 0
View file @ 6425c3c2
[seed],[wsp],[wave_seed],[wdir],[ReferenceTI],[ReferenceWindSpeed],[t0],[duration],[hub_height],[Case folder],[Duration],[Case id.],[Turb base name],[time_stop]
1001,4,101,10,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp04_wdir010_sa1001_sw0101,turb_wsp04_s1001,3700
1001,4,101,20,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp04_wdir020_sa1001_sw0101,turb_wsp04_s1001,3700
1001,4,102,10,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp04_wdir010_sa1001_sw0102,turb_wsp04_s1001,3700
2001,4,102,20,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp04_wdir020_sa2001_sw0102,turb_wsp04_s2001,3700
2001,4,103,10,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp04_wdir010_sa2001_sw0103,turb_wsp04_s2001,3700
2001,4,103,20,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp04_wdir020_sa2001_sw0103,turb_wsp04_s2001,3700
3002,6,201,10,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp06_wdir010_sa3002_sw0201,turb_wsp06_s3002,3700
3002,6,201,20,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp06_wdir020_sa3002_sw0201,turb_wsp06_s3002,3700
3002,6,202,10,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp06_wdir010_sa3002_sw0202,turb_wsp06_s3002,3700
4002,6,202,20,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp06_wdir020_sa4002_sw0202,turb_wsp06_s4002,3700
4002,6,203,10,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp06_wdir010_sa4002_sw0203,turb_wsp06_s4002,3700
4002,6,203,20,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp06_wdir020_sa4002_sw0203,turb_wsp06_s4002,3700
5003,8,301,10,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp08_wdir010_sa5003_sw0301,turb_wsp08_s5003,3700
5003,8,301,20,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp08_wdir020_sa5003_sw0301,turb_wsp08_s5003,3700
5003,8,302,10,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp08_wdir010_sa5003_sw0302,turb_wsp08_s5003,3700
6003,8,302,20,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp08_wdir020_sa6003_sw0302,turb_wsp08_s6003,3700
6003,8,303,10,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp08_wdir010_sa6003_sw0303,turb_wsp08_s6003,3700
6003,8,303,20,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp08_wdir020_sa6003_sw0303,turb_wsp08_s6003,3700
7001,4,101,10,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp04_wdir010_sa7001_sw0101,turb_wsp04_s7001,3700
7001,4,101,20,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp04_wdir020_sa7001_sw0101,turb_wsp04_s7001,3700
7001,4,102,10,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp04_wdir010_sa7001_sw0102,turb_wsp04_s7001,3700
8001,4,102,20,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp04_wdir020_sa8001_sw0102,turb_wsp04_s8001,3700
8001,4,103,10,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp04_wdir010_sa8001_sw0103,turb_wsp04_s8001,3700
8001,4,103,20,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp04_wdir020_sa8001_sw0103,turb_wsp04_s8001,3700
9002,6,201,10,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp06_wdir010_sa9002_sw0201,turb_wsp06_s9002,3700
9002,6,201,20,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp06_wdir020_sa9002_sw0201,turb_wsp06_s9002,3700
9002,6,202,10,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp06_wdir010_sa9002_sw0202,turb_wsp06_s9002,3700
10002,6,202,20,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp06_wdir020_sa10002_sw0202,turb_wsp06_s10002,3700
10002,6,203,10,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp06_wdir010_sa10002_sw0203,turb_wsp06_s10002,3700
10002,6,203,20,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp06_wdir020_sa10002_sw0203,turb_wsp06_s10002,3700
11003,8,301,10,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp08_wdir010_sa11003_sw0301,turb_wsp08_s11003,3700
11003,8,301,20,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp08_wdir020_sa11003_sw0301,turb_wsp08_s11003,3700
11003,8,302,10,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp08_wdir010_sa11003_sw0302,turb_wsp08_s11003,3700
12003,8,302,20,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp08_wdir020_sa12003_sw0302,turb_wsp08_s12003,3700
12003,8,303,10,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp08_wdir010_sa12003_sw0303,turb_wsp08_s12003,3700
12003,8,303,20,0.12,44,100,100,116.8,DLC12,3600,DLC12_wsp08_wdir020_sa12003_sw0303,turb_wsp08_s12003,3700
wetb/prepost/tests/data/demo_gendlc/ref/DLC13.csv 0 → 100644
+ 25
− 0
View file @ 6425c3c2
[seed],[wave_seed],[wsp],[wdir],[ReferenceTI],[ReferenceWindSpeed],[t0],[duration],[hub_height],[Case folder],[Duration],[Case id.],[Turb base name],[time_stop]
1001,101,4,0,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp04_wdir000_sa1001_sw0101,turb_wsp04_s1001,3700
1001,101,4,20,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp04_wdir020_sa1001_sw0101,turb_wsp04_s1001,3700
1002,201,6,0,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp06_wdir000_sa1002_sw0201,turb_wsp06_s1002,3700
2002,201,6,20,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp06_wdir020_sa2002_sw0201,turb_wsp06_s2002,3700
2003,301,8,0,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp08_wdir000_sa2003_sw0301,turb_wsp08_s2003,3700
2003,301,8,20,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp08_wdir020_sa2003_sw0301,turb_wsp08_s2003,3700
3001,102,4,0,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp04_wdir000_sa3001_sw0102,turb_wsp04_s3001,3700
3001,102,4,20,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp04_wdir020_sa3001_sw0102,turb_wsp04_s3001,3700
3002,202,6,0,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp06_wdir000_sa3002_sw0202,turb_wsp06_s3002,3700
4002,202,6,20,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp06_wdir020_sa4002_sw0202,turb_wsp06_s4002,3700
4003,302,8,0,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp08_wdir000_sa4003_sw0302,turb_wsp08_s4003,3700
4003,302,8,20,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp08_wdir020_sa4003_sw0302,turb_wsp08_s4003,3700
5001,101,4,0,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp04_wdir000_sa5001_sw0101,turb_wsp04_s5001,3700
5001,101,4,20,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp04_wdir020_sa5001_sw0101,turb_wsp04_s5001,3700
5002,201,6,0,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp06_wdir000_sa5002_sw0201,turb_wsp06_s5002,3700
6002,201,6,20,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp06_wdir020_sa6002_sw0201,turb_wsp06_s6002,3700
6003,301,8,0,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp08_wdir000_sa6003_sw0301,turb_wsp08_s6003,3700
6003,301,8,20,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp08_wdir020_sa6003_sw0301,turb_wsp08_s6003,3700
7001,102,4,0,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp04_wdir000_sa7001_sw0102,turb_wsp04_s7001,3700
7001,102,4,20,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp04_wdir020_sa7001_sw0102,turb_wsp04_s7001,3700
7002,202,6,0,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp06_wdir000_sa7002_sw0202,turb_wsp06_s7002,3700
8002,202,6,20,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp06_wdir020_sa8002_sw0202,turb_wsp06_s8002,3700
8003,302,8,0,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp08_wdir000_sa8003_sw0302,turb_wsp08_s8003,3700
8003,302,8,20,0.12,44,100,100,116.8,DLC13,3600,DLC13_wsp08_wdir020_sa8003_sw0302,turb_wsp08_s8003,3700
wetb/prepost/tests/test_GenerateDLCs.py 0 → 100644
+ 60
− 0
View file @ 6425c3c2
from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
from future import standard_library
standard_library.install_aliases()
import unittest
import os
#import shutil
import numpy as np
import pandas as pd
from wetb.prepost.GenerateDLCs import GenerateDLCCases
class Template(unittest.TestCase):
def setUp(self):
self.basepath = os.path.dirname(__file__)
class TestGenerateDLCCases(Template):
def test_dlcs(self):
# manually configure paths, HAWC2 model root path is then constructed as
# p_root_remote/PROJECT/sim_id, and p_root_local/PROJECT/sim_id
# adopt accordingly when you have configured your directories differently
p_root = os.path.join(self.basepath, 'data/demo_gendlc/')
# project name, sim_id, master file name
dlc_master = os.path.join(p_root, 'DLCs.xlsx')
dlc_folder = os.path.join(p_root, 'DLCs')
dlc_gen12 = os.path.join(dlc_folder, 'DLC12.xlsx')
dlc_gen13 = os.path.join(dlc_folder, 'DLC13.xlsx')
DLB = GenerateDLCCases()
DLB.execute(filename=dlc_master, folder=dlc_folder)
df12 = pd.read_excel(dlc_gen12)
# df12.to_csv('data/demo_gendlc/ref/DLC12.csv', index=False)
df12_ref = pd.read_csv(os.path.join(p_root, 'ref/DLC12.csv'))
# df12_ref2 = pd.read_excel(p2)[df12.columns]
pd.testing.assert_frame_equal(df12, df12_ref)
# df2 = df[['[Case id.]', '[wdir]', '[wsp]', '[seed]', '[wave_seed]']]
self.assertEqual(df12['[ReferenceWindSpeed]'].unique(), np.array([44]))
self.assertEqual(df12['[t0]'].unique(), np.array([100]))
self.assertEqual(len(df12['[Case id.]'].unique()), 2*3*3*2)
self.assertEqual(df12['[Case id.]'].values[0],
'DLC12_wsp04_wdir010_sa1001_sw0101')
df13 = pd.read_excel(dlc_gen13)
# df13.to_csv('data/demo_gendlc/ref/DLC13.csv', index=False)
df13_ref = pd.read_csv(os.path.join(p_root, 'ref/DLC13.csv'))
pd.testing.assert_frame_equal(df13, df13_ref)
if __name__ == "__main__":
unittest.main()
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment