Commit d074c620 authored by Mikkel Friis-Møller's avatar Mikkel Friis-Møller
Browse files

remote mongo for srv-type uri's

parent e8645fd2
Pipeline #31788 passed with stages
in 21 minutes and 10 seconds
......@@ -242,7 +242,10 @@
" constraints=[SpacingConstraint(5*D), \n",
" XYBoundaryConstraint(boundary=boundary, boundary_type='polygon')],\n",
" plot_comp=XYPlotComp(),\n",
" recorder=MongoRecorder(case_id='test', clean_up=True)\n",
" recorder=MongoRecorder(uri = \"mongodb+srv://TopfarmUser:lstN6RRM5M0XSzaF@topfarm.20e5l.mongodb.net/data22?retryWrites=true&w=majority\",\n",
" db_name='data22',\n",
" case_id='test', \n",
" clean_up=True)\n",
" )\n",
"\n",
"# run state before optimize\n",
......@@ -275,7 +278,7 @@
"Optimization FAILED.\n",
"Iteration limit reached\n",
"-----------------------------------\n",
"Optimized in\t51.847s\n"
"Optimized in\t60.464s\n"
]
},
{
......@@ -333,7 +336,7 @@
{
"data": {
"text/plain": [
"[<matplotlib.lines.Line2D at 0x2304527b790>]"
"[<matplotlib.lines.Line2D at 0x1594e410f40>]"
]
},
"execution_count": 9,
......
%% Cell type:markdown id:65a0003c tags:
# Mongo DB demonstration
Example on how to use the MongoRecorder as the main recorder to store data in a local database.
%% Cell type:markdown id:a6407ebc tags:
[Try this yourself](https://colab.research.google.com/github/DTUWindEnergy/TopFarm2/blob/master/docs/notebooks/MongoDB_recorder.ipynb) (requires google account)
%% Cell type:code id:7809feca tags:
``` python
# Install TopFarm if needed
import importlib
if not importlib.util.find_spec("topfarm"):
!pip install git+https://gitlab.windenergy.dtu.dk/TOPFARM/TopFarm2.git
```
%% Cell type:code id:4f357cff tags:
``` python
# Import dependencies
import numpy as np
import matplotlib.pyplot as plt
plt.rcParams['figure.dpi'] = 150
from py_wake.site.xrsite import GlobalWindAtlasSite
from py_wake.deficit_models.gaussian import IEA37SimpleBastankhahGaussian
from py_wake.examples.data.hornsrev1 import V80
from topfarm.constraint_components.boundary import XYBoundaryConstraint
from topfarm.constraint_components.spacing import SpacingConstraint
from topfarm.cost_models.cost_model_wrappers import CostModelComponent, AEPCostModelComponent
from topfarm import TopFarmGroup, TopFarmProblem
from topfarm.plotting import XYPlotComp
from topfarm.easy_drivers import EasyScipyOptimizeDriver
from topfarm.mongo_recorder import MongoRecorder
import subprocess
subprocess.Popen(['mongod'])
```
%%%% Output: stream
C:\Users\mikf\Anaconda3\envs\om3\lib\site-packages\openmdao\utils\general_utils.py:128: OMDeprecationWarning:simple_warning is deprecated. Use openmdao.utils.om_warnings.issue_warning instead.
C:\Users\mikf\Anaconda3\envs\om3\lib\site-packages\openmdao\utils\notebook_utils.py:157: UserWarning:Tabulate is not installed. Run `pip install openmdao[notebooks]` to install required dependencies. Using ASCII for outputs.
%%%% Output: execute_result
<Popen: returncode: None args: ['mongod']>
%% Cell type:markdown id:7b0e2bbe tags:
The site is given by the GlobalWindAtlasSite function through the latitude and longitude coordinates. The used wind turbine model is the V80. A number of 24 wind turbines are displayed in a pattern likewise the one used at Hornsrev1.
%% Cell type:code id:d7adb995 tags:
``` python
# Site and wind turbine definition
wt = V80()
D = wt.diameter()
hub_height = wt.hub_height()
lat, long = 44.001508, -8.200195
height = hub_height
roughness = 0.05
site = GlobalWindAtlasSite(lat, long, height, roughness, ti=0.5)
```
%% Cell type:code id:0866071e tags:
``` python
# Layout coordinates
xy = np.array([[423974, 6151447],
[424042, 6150891],
[424111, 6150335],
[424179, 6149779],
[424247, 6149224],
[424315, 6148668],
[424384, 6148112],
[424452, 6147556],
[424534, 6151447],
[424602, 6150891],
[424671, 6150335],
[424739, 6149779],
[424807, 6149224],
[424875, 6148668],
[424944, 6148112],
[425012, 6147556],
[425094, 6151447],
[425162, 6150891],
[425231, 6150335],
[425299, 6149779],
[425367, 6149224],
[425435, 6148668],
[425504, 6148112],
[425572, 6147556]])
# site boundaries
boundary = [(423500, 6.1474e6), (425700, 6.1474e6),
(425200, 6.1515e6), (423000, 6.1515e6)]
```
%% Cell type:markdown id:60760260 tags:
The cost function is the AEP of the wind farm. The wake model used is the IEA37 Simple Bastankhah Gaussian, predefined in PyWake. The cost component is fed with the layout coordinates 'x' and 'y'.
%% Cell type:code id:32cee548 tags:
``` python
def aep_func(x, y, **kwargs):
wake_model = IEA37SimpleBastankhahGaussian(site, wt)
simres = wake_model(x, y)
aep = simres.aep().sum()
return aep
# create an openmdao component for aep
aep_comp = CostModelComponent(input_keys=['x', 'y'],
n_wt=len(xy),
cost_function=aep_func,
output_key="aep",
output_unit="GWh",
objective=True,
output_val=sum(np.zeros(len(xy))),
maximize=True
)
```
%%%% Output: stream
<ipython-input-5-5fcb277fbf00>:9: DeprecationWarning: output_key is deprecated; use keyword output_keys instead
aep_comp = CostModelComponent(input_keys=['x', 'y'],
<ipython-input-5-5fcb277fbf00>:9: DeprecationWarning: output_val is deprecated; use keyword output_vals instead
aep_comp = CostModelComponent(input_keys=['x', 'y'],
%% Cell type:markdown id:b509e6a7 tags:
## Definition of the Topfarm Problem
In order to make it more flexible, we have implemented a new option within the TopFarmProblem object that allows to choose the recorder. By default, the TopFarmListRecorder is used. The case_id variable defines the ID of the simulation under which it is stored in the Mongo database.
%% Cell type:code id:c26b1d0b tags:
``` python
# define problem object
problem = TopFarmProblem(design_vars={'x': xy[:, 0], 'y': xy[:, 1]},
cost_comp=aep_comp,
driver=EasyScipyOptimizeDriver(optimizer='SLSQP', maxiter=10, tol=1e-6),
constraints=[SpacingConstraint(5*D),
XYBoundaryConstraint(boundary=boundary, boundary_type='polygon')],
plot_comp=XYPlotComp(),
recorder=MongoRecorder(case_id='test', clean_up=True)
recorder=MongoRecorder(uri = "mongodb+srv://TopfarmUser:lstN6RRM5M0XSzaF@topfarm.20e5l.mongodb.net/data22?retryWrites=true&w=majority",
db_name='data22',
case_id='test',
clean_up=True)
)
# run state before optimize
cost, state = problem.evaluate()
```
%%%% Output: stream
INFO: checking out_of_order
INFO: checking system
INFO: checking solvers
INFO: checking dup_inputs
INFO: checking missing_recorders
INFO: checking unserializable_options
INFO: checking comp_has_no_outputs
INFO: checking auto_ivc_warnings
INFO: checking out_of_order
INFO: checking system
INFO: checking solvers
INFO: checking dup_inputs
INFO: checking missing_recorders
INFO: checking unserializable_options
INFO: checking comp_has_no_outputs
INFO: checking auto_ivc_warnings
%%%% Output: display_data
%% Cell type:code id:5dd091fb tags:
``` python
# Optimize
cost, state, recorder = problem.optimize(disp=True)
```
%%%% Output: stream
INFO: checking out_of_order
INFO: checking system
INFO: checking solvers
INFO: checking dup_inputs
INFO: checking missing_recorders
INFO: checking unserializable_options
INFO: checking comp_has_no_outputs
INFO: checking auto_ivc_warnings
Iteration limit reached (Exit mode 9)
Current function value: [-143.912072]
Iterations: 10
Function evaluations: 10
Gradient evaluations: 10
Optimization FAILED.
Iteration limit reached
-----------------------------------
Optimized in 51.847s
Optimized in 60.464s
%%%% Output: display_data
%% Cell type:markdown id:df31ffb7 tags:
## How to access the database from TopFarm
Accessing is the database is done in the same way as the TopfarmListRecorder:
%% Cell type:code id:6e14def9 tags:
``` python
keys = recorder.keys()
print(keys)
```
%%%% Output: stream
['_id', 'case_id', 'run_id', 'counter', 'timestamp', 'success', 'msg', 'rank', 'size', 'max_step', 'cost', 'penalty', 'x', 'y', 'penalty_spacing_comp_400', 'penalty_xyboundary_comp_polygon_26295200', 'aep', 'aggr_cost', 'cost_comp_eval', 'xy_boundary', 'plot_counter', 'wtSeparationSquared', 'boundaryDistances']
%% Cell type:code id:a7370b66 tags:
``` python
aep = recorder['aep']
plt.plot(aep)
```
%%%% Output: execute_result
[<matplotlib.lines.Line2D at 0x2304527b790>]
[<matplotlib.lines.Line2D at 0x1594e410f40>]
%%%% Output: display_data
%% Cell type:markdown id:85229e7b tags:
Animate recording and display.
%% Cell type:code id:812da2e8 tags:
``` python
# The code in this cell will not work in VSCode, due to webview beeing blocked by Microsoft.
# an alternate option for VSCode users are to save the animation and view in a mediaplayer.
# see cell below.
# from IPython.display import HTML
# anim = recorder.animate_turbineXY(duration=10, tail=5, cost='aep', anim_options = {'interval': 20, 'blit': True})
# html = HTML(anim.to_html5_video())
# display(html)
# plt.close()
```
%% Cell type:code id:e882e928 tags:
``` python
# anim = recorder.animate_turbineXY(filename='animation')
# plt.close()
```
%% Cell type:code id:4d9034ab tags:
``` python
```
......
......@@ -24,4 +24,5 @@ You can also `launch them via Binder <https://mybinder.org/v2/git/https%3A%2F%2F
examples_nblinks/layout_and_loads_nb
examples_nblinks/bathymetry_nb
examples_nblinks/exclusion_zones_nb
examples_nblinks/mongodb_nb
{
"path": "../../notebooks/MongoDB_recorder.ipynb"
}
\ No newline at end of file
......@@ -34,7 +34,7 @@ setup(name='topfarm',
'pycodestyle', # for testing
'pytest-cov', # for calculating coverage
'py_wake>=2', # for calculating AEP
'pymongo', # for mongo_recorder
'pymongo[srv]', # for mongo_recorder
'scipy', # constraints
'sphinx', # generating documentation
'sphinx_rtd_theme', # docs theme
......
......@@ -94,18 +94,33 @@ def clean_keys(o):
class RemoteMongo():
def __init__(self, port=27017, ip='localhost', password=None, username=None, db_name='database'):
def __init__(self, uri=None, host=None, port=27017, ip='localhost', password=None, user=None, db_name='database', uri_type=None, **kwargs):
self.host = host
self.port = port
self.ip = ip
self.username = username
self.uri = uri
self.user = user
self.password = password
self.db_name = db_name
self.uri_type = uri_type
self.kwargs = kwargs
def open(self):
self.client = MongoClient(f'mongodb://{self.ip}:{self.port}',
username=self.username,
password=self.password,
authSource='database_str')
if self.uri:
uri = self.uri
self.kwargs.update({'host': uri})
elif self.host:
if not self.uri_type:
uri = f"mongodb://{self.user}:{self.password}@{self.host}"
elif self.uri_type == 'srv':
uri = f"mongodb+srv://{self.user}:{self.password}@{self.host}"
else:
self.kwargs.update({'host': f'mongodb://{self.ip}:{self.port}',
'username': self.user,
'password': self.password,
'authSource': 'database_str'})
self.client = MongoClient(**self.kwargs)
self.db = self.client[self.db_name]
return self.client
......@@ -143,8 +158,8 @@ class MongoRecorder(CaseRecorder):
Flag indicating whether to record on this processor when running in parallel.
"""
def __init__(self, ip='localhost', port=27017, record_viewer_data=True, ssh_tunnel=False,
db_name='database', case_id=None, clean_up=False, with_mpi=False):
def __init__(self, uri=None, host=None, ip='localhost', port=27017, user=None, password=None, uri_type=None, record_viewer_data=True, ssh_tunnel=False,
db_name='database', case_id=None, clean_up=False, with_mpi=False, client_args={}):
"""
Initialize the MongoRecorder.
......@@ -159,6 +174,12 @@ class MongoRecorder(CaseRecorder):
"""
self._ip = ip
self._port = port
self._host = host
self._uri = uri
self._user = user
self._password = password
self._uri_type = uri_type
self._client_args = client_args
self.case_id = case_id
self.db_name = db_name
self.ssh_tunnel = ssh_tunnel
......@@ -258,7 +279,7 @@ class MongoRecorder(CaseRecorder):
print('Warning: SSH database access not available in this Topfarm version!')
# return EC_Mongo()
else:
client = RemoteMongo(port=self._port, ip=self._ip, db_name=self.db_name)
client = RemoteMongo(uri=self._uri, host=self._host, port=self._port, ip=self._ip, user=self._user, password=self._password, db_name=self.db_name, uri_type=self._uri_type, **self._client_args)
return client
def exists(self):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment