Commit b7e748c0 authored by Mikkel Friis-Møller's avatar Mikkel Friis-Møller
Browse files

enabled ssh connection to jess instead of gorm as this is dead

parent 4d0b36fb
Pipeline #6423 passed with stage
in 36 seconds
......@@ -6,14 +6,15 @@ if __name__ == '__main__':
try:
import password
user = password.user
password = password.pass_gorm
password = password.pass_jess
except:
user = input('Username: ')
password = input('Password: ')
calc_dir = os.path.join(os.path.dirname(__file__), 'example_3')
v52 = V52(calc_dir)
v52.get_model()
sim_resource = 2
if sim_resource == 2:
v52.copy_model_to_cluster(user, password)
v52.run_simulation(user, password, sim_resource)
v52.copy_model_to_cluster(user, password,host='jess.dtu.dk')
# v52.run_simulation(user, password, sim_resource)
import mysql.connector
import pandas as pd
from sqlalchemy import create_engine
from datetime import datetime
try:
import MySQLdb
def SQLconnect(I):
cnx = MySQLdb.connect(
host = I['host'],
user = I['user'],
passwd = I['password'],
db = I['database'])
return cnx
except:
import mysql.connector
def SQLconnect(I):
cnx = mysql.connector.connect(
user=I['user'],
password=I['password'],
host=I['host'],
database=I['database'])
return cnx
def SQLconnect(I):
cnx = mysql.connector.connect(
user=I['user'],
password=I['password'],
host=I['host'],
database=I['database'])
return cnx
def SQLdataframe(cnx, table_name, limit='default', col='*', row='1=1', logical_statements=[]):
......@@ -321,6 +331,7 @@ if __name__ == '__main__':
'host': '10.40.20.14',
}
cnx = SQLconnect(I)
# cnx = SQLconnect2(I)
# string = "SELECT * FROM caldata_2018_03_50hz WHERE `Name` LIKE '201803010100'"
# startTime = datetime.now()
......@@ -354,4 +365,45 @@ if __name__ == '__main__':
## 0:01:10.284159
# string = "SELECT * FROM caldata_2018_03_50hz WHERE `Name` LIKE '201803010100' OR `Name` LIKE '201803010110' OR `Name` LIKE '201803010120'"
# startTime = datetime.now()
# mycursor = cnx.cursor()
# mycursor.execute(string)
# myresult = mycursor.fetchall()
# print(datetime.now() - startTime)
## 0:00:56.604824
# string = "SELECT * FROM caldata_2018_03_50hz WHERE `Name` >= '201803010100' AND `Name` <= '201803010120'"
# startTime = datetime.now()
# mycursor = cnx.cursor()
# mycursor.execute(string)
# myresult = mycursor.fetchall()
# print(datetime.now() - startTime)
## 0:00:54.983104
# string = "SELECT * FROM caldata_2018_03_50hz WHERE `Name` >= '201803010100' AND `Name` <= '201803010120'"
# cnx = SQLconnect2(I)
# startTime = datetime.now()
# cursor = cnx.cursor()
# cursor.execute (string)
# myresult = cursor.fetchall()
# print(datetime.now() - startTime)
## 0:00:36.591643
# string = "SELECT * FROM caldata_2018_03_50hz WHERE `Name` LIKE '201803010100' OR `Name` LIKE '201803010110' OR `Name` LIKE '201803010120'"
# startTime = datetime.now()
# cursor = cnx.cursor()
# cursor.execute (string)
# myresult = cursor.fetchall()
# print(datetime.now() - startTime)
## 0:00:35.850298
startTime = datetime.now()
string = "SELECT * FROM caldata_2018_03_50hz WHERE `Name` LIKE '201803010100' OR `Name` LIKE '201803010110' OR `Name` LIKE '201803010120'"
df = pd.read_sql(
string,
con=cnx
)
print(datetime.now() - startTime)
......@@ -3,6 +3,7 @@ from v52.utils import (get_time_series, get_data_from_SQL, stat_val, bin_data,
dump_data, get_data_from_file, ntf, get_model,
copy_to_calc_dir, transfer_model, load_sel)
from v52.plot import plot_windrose, plot
from v52.create_turb import create_turb
import os
from shutil import copy
from wetb.hawc2.htc_file import HTCFile
......@@ -178,15 +179,15 @@ class V52(object):
def copy_model_to_cluster(self,username, password):
def copy_model_to_cluster(self,username, password, host='gorm.risoe.dk'):
localpath = self.hawc2_dir
remotepath = r'wine_exe/win32/'
transfer_model(username, password, localpath, remotepath)
transfer_model(username, password, localpath, remotepath, host)
folders = ['data','control']
for folder in folders:
lp = os.path.join(localpath,folder)
rp = os.path.join(remotepath,folder)
transfer_model(username, password, lp,rp)
transfer_model(username, password, lp,rp, host)
......@@ -351,6 +352,12 @@ class V52(object):
dic[LC['name']] = [x[:-4] for x in files]
self.info['LC_files'] = dic
return dic
def create_turb(self, turb_gen, lc, turb_type='mann'):
LCs = self.info['LCs']
for LC in LCs:
create_turb(self.hawc2_dir, turb_gen, LC['name'],
turb_type=turb_type)
if __name__ == '__main__':
......
import os
import pickle
from wetb.hawc2.htc_file import HTCFile
def create_turb(hawc2_dir, turb_gen, lc, turb_type=None):
if not turb_type:
turb_type='mann'
htc_dir = os.path.join(hawc2_dir,'htc')
turb_dir = os.path.join(hawc2_dir, 'turb', lc)
if not os.path.exists(turb_dir):
os.makedirs(turb_dir)
for file in os.listdir(os.path.join(htc_dir,lc)):
ID = file[:-4]
turb_path = os.path.join(turb_dir, ID + '_')
htc_path = os.path.join(htc_dir,lc,file)
if not (os.path.exists(turb_path + 'u.bin') and
os.path.exists(turb_path + 'v.bin') and
os.path.exists(turb_path + 'w.bin')):
turb_dic = get_turb_params_from_htc(htc_path)
if turb_type=='mann':
create_mann_turb(turb_gen, turb_dic, turb_path)
def create_mann_turb(csimu2_path, turb_dic, turb_path):
csimu2_string = csimu2_path + ' '
csimu2_string += turb_dic['turb_string']
csimu2_string += ' ' + turb_path
csimu2_string = csimu2_string.lower()
print(csimu2_string)
os.system(csimu2_string)
def create_ksec_turb(turb_dic, dic, turb_path, inp, turb_dir):
turb_constr = inp['turb_constr']
ksec_simu_path = inp['ksec_simu_path']
ny = turb_dic['ny']
nz = turb_dic['nz']
sy = turb_dic['sy']
sz = turb_dic['sz']
zc = 44 # Hardcoded for V52
df = dic['meas_stats']
mast_pickle_path = turb_path + 'mast.p'
ksec_bat_path = turb_path + 'ksec.bat'
with open(mast_pickle_path, 'wb') as g:
pickle.dump(df, g)
if turb_constr == 0:
con_type = 'unc'
elif turb_constr == 1:
con_type = 'con'
else:
raise ValueError(f'Unrecognized simulation type "{con_type}"')
ksec_string = 'python {} {} {} {} {} {} {} {} {}'.format(
ksec_simu_path, ny, nz, sy, sz, zc, con_type, mast_pickle_path,
turb_dir)
with open(ksec_bat_path, 'w') as g:
g.write(ksec_string)
os.system(ksec_string)
def get_turb_params_from_htc(htc_path):
htc = HTCFile(htc_path)
nx, dx = htc.wind.mann.box_dim_u[:]
ny, dy = htc.wind.mann.box_dim_v[:]
nz, dz = htc.wind.mann.box_dim_w[:]
sx, sy, sz = [(nx-1)*dx, (ny-1)*dy, (nz-1)*dz]
L, ae, Gamma, seed, highfrq_comp = htc.wind.mann.create_turb_parameters[:]
turb_lst = [nx, ny, nz, sx, sy, sz, ae, L, Gamma, seed]
turb_lst = [str(x) for x in turb_lst]
turb_string = ' '.join(turb_lst)
turb_dic = {
'nx': nx,
'ny': ny,
'nz': nz,
'dx': dx,
'dy': dy,
'dz': dz,
'sx': sx,
'sy': sy,
'sz': sz,
'ae': ae,
'L': L,
'Gamma': Gamma,
'seed': seed,
'turb_string': turb_string,
}
return turb_dic
if __name__ == '__main__':
hawc2_dir = r'C:\Sandbox\Git\V52\tjul\HAWC2'
csimu2_path = r"C:\Sandbox\Git\V52\tjul\csimu2.exe"
lc = 'sa_wsp_sa_ti'
create_turb(hawc2_dir, csimu2_path, lc)
......@@ -3,8 +3,157 @@ from v52.utils import get_time_series
start = (2018, 3, 1, 1, 10)
stop = (2018, 3, 2, 2, 20)
ref = ['201803010110', '201803010120', '201803010130', '201803010140', '201803010150', '201803010210', '201803010220', '201803010230', '201803010240', '201803010250', '201803010310', '201803010320', '201803010330', '201803010340', '201803010350', '201803010410', '201803010420', '201803010430', '201803010440', '201803010450', '201803010510', '201803010520', '201803010530', '201803010540', '201803010550', '201803010610', '201803010620', '201803010630', '201803010640', '201803010650', '201803010710', '201803010720', '201803010730', '201803010740', '201803010750', '201803010810', '201803010820', '201803010830', '201803010840', '201803010850', '201803010910', '201803010920', '201803010930', '201803010940', '201803010950', '201803011010', '201803011020', '201803011030', '201803011040', '201803011050', '201803011110', '201803011120', '201803011130', '201803011140', '201803011150', '201803011210', '201803011220', '201803011230', '201803011240', '201803011250', '201803011310', '201803011320', '201803011330', '201803011340', '201803011350', '201803011410', '201803011420', '201803011430', '201803011440', '201803011450', '201803011510', '201803011520', '201803011530', '201803011540', '201803011550', '201803011610', '201803011620', '201803011630', '201803011640', '201803011650', '201803011710', '201803011720', '201803011730', '201803011740', '201803011750', '201803011810', '201803011820', '201803011830', '201803011840', '201803011850', '201803011910', '201803011920', '201803011930', '201803011940', '201803011950', '201803012010', '201803012020', '201803012030', '201803012040', '201803012050', '201803012110', '201803012120', '201803012130', '201803012140', '201803012150', '201803012210', '201803012220', '201803012230', '201803012240', '201803012250', '201803012310', '201803012320', '201803012330', '201803012340', '201803012350', '201803020000', '201803020010', '201803020100', '201803020110']
ref = ['201803010110',
'201803010120',
'201803010130',
'201803010140',
'201803010150',
'201803010200',
'201803010210',
'201803010220',
'201803010230',
'201803010240',
'201803010250',
'201803010300',
'201803010310',
'201803010320',
'201803010330',
'201803010340',
'201803010350',
'201803010400',
'201803010410',
'201803010420',
'201803010430',
'201803010440',
'201803010450',
'201803010500',
'201803010510',
'201803010520',
'201803010530',
'201803010540',
'201803010550',
'201803010600',
'201803010610',
'201803010620',
'201803010630',
'201803010640',
'201803010650',
'201803010700',
'201803010710',
'201803010720',
'201803010730',
'201803010740',
'201803010750',
'201803010800',
'201803010810',
'201803010820',
'201803010830',
'201803010840',
'201803010850',
'201803010900',
'201803010910',
'201803010920',
'201803010930',
'201803010940',
'201803010950',
'201803011000',
'201803011010',
'201803011020',
'201803011030',
'201803011040',
'201803011050',
'201803011100',
'201803011110',
'201803011120',
'201803011130',
'201803011140',
'201803011150',
'201803011200',
'201803011210',
'201803011220',
'201803011230',
'201803011240',
'201803011250',
'201803011300',
'201803011310',
'201803011320',
'201803011330',
'201803011340',
'201803011350',
'201803011400',
'201803011410',
'201803011420',
'201803011430',
'201803011440',
'201803011450',
'201803011500',
'201803011510',
'201803011520',
'201803011530',
'201803011540',
'201803011550',
'201803011600',
'201803011610',
'201803011620',
'201803011630',
'201803011640',
'201803011650',
'201803011700',
'201803011710',
'201803011720',
'201803011730',
'201803011740',
'201803011750',
'201803011800',
'201803011810',
'201803011820',
'201803011830',
'201803011840',
'201803011850',
'201803011900',
'201803011910',
'201803011920',
'201803011930',
'201803011940',
'201803011950',
'201803012000',
'201803012010',
'201803012020',
'201803012030',
'201803012040',
'201803012050',
'201803012100',
'201803012110',
'201803012120',
'201803012130',
'201803012140',
'201803012150',
'201803012200',
'201803012210',
'201803012220',
'201803012230',
'201803012240',
'201803012250',
'201803012300',
'201803012310',
'201803012320',
'201803012330',
'201803012340',
'201803012350',
'201803020000',
'201803020010',
'201803020020',
'201803020030',
'201803020040',
'201803020050',
'201803020100',
'201803020110',
'201803020120',
'201803020130',
'201803020140',
'201803020150',
'201803020200',
'201803020210']
def test_get_time_series():
lst = get_time_series(start, stop)
......
......@@ -35,13 +35,19 @@ def get_time_series(start, stop):
hh = hh2
lst.append(_ts_string(YYYY, MM, DD, hh, mm))
elif i == 0:
for hh in range(hh1, 24):
for mm in range(int(mm1/10), 6):
for mm in range(int(mm1/10), 6):
hh = hh1
lst.append(_ts_string(YYYY, MM, DD, hh, mm))
for hh in range(hh1+1, 24):
for mm in range(0, 6):
lst.append(_ts_string(YYYY, MM, DD, hh, mm))
elif i == delta.days:
for hh in range(0, hh2):
for mm in range(0, int(mm2/10)):
for mm in range(0, 6):
lst.append(_ts_string(YYYY, MM, DD, hh, mm))
for mm in range(0, int(mm2/10)):
hh = hh2
lst.append(_ts_string(YYYY, MM, DD, hh, mm))
else:
for hh in range(0, 24):
for mm in range(0, 6):
......@@ -203,8 +209,8 @@ def copy_to_calc_dir(repo_dir, hawc2_dir, htc_template_dir, res_dir,
os.makedirs(res_dir)
def transfer_model(username, password, localpath, remotepath):
client = SSHClient(host='gorm.risoe.dk', port=22, username=username, password=password)
def transfer_model(username, password, localpath, remotepath, host='gorm.risoe.dk'):
client = SSHClient(host=host, port=22, username=username, password=password)
file_lst = []
for file in os.listdir(localpath):
if os.path.isfile(os.path.join(localpath, file)):
......@@ -230,8 +236,8 @@ def load_sel(sel_path):
if __name__ == '__main__':
start = (2018, 9, 15, 9, 50)
stop = (2018, 9, 15, 12, 10)
start = (2018, 10, 11, 0, 0)
stop = (2018, 10, 12, 8, 0)
lst = get_time_series(start, stop)
print(lst)
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment