diff --git a/wetb/prepost/dlctemplate.py b/wetb/prepost/dlctemplate.py index 81f723c0e83c702dd4bf509b2ed58fb5275b563c..c697787353db8d8d64a1874d27971362c4f24756 100644 --- a/wetb/prepost/dlctemplate.py +++ b/wetb/prepost/dlctemplate.py @@ -265,7 +265,8 @@ def variable_tag_func_mod1(master, case_id_short=False): def launch_dlcs_excel(sim_id, silent=False, verbose=False, pbs_turb=False, runmethod=None, write_htc=True, zipchunks=False, - walltime='04:00:00', postpro_node=False): + walltime='04:00:00', postpro_node=False, + dlcs_dir='htc/DLCs'): """ Launch load cases defined in Excel files """ @@ -279,7 +280,6 @@ def launch_dlcs_excel(sim_id, silent=False, verbose=False, pbs_turb=False, pyenv = None # see if a htc/DLCs dir exists - dlcs_dir = os.path.join(P_SOURCE, 'htc', 'DLCs') # Load all DLC definitions and make some assumptions on tags that are not # defined if os.path.exists(dlcs_dir): @@ -458,11 +458,14 @@ def post_launch(sim_id, statistics=True, rem_failed=True, check_logs=True, return df_stats, df_AEP, df_Leq -def postpro_node_merge(tqdm=False): +def postpro_node_merge(tqdm=False, zipchunks=False): """With postpro_node each individual case has a .csv file for the log file analysis and a .csv file for the statistics tables. Merge all these single files into one table/DataFrame. + When using the zipchunk approach, all log file analysis and statistics + are grouped into tar archives in the prepost-data directory. + Parameters ---------- @@ -470,12 +473,18 @@ def postpro_node_merge(tqdm=False): Set to True for displaying a progress bar (provided by the tqdm module) when merging all csv files into a single table/pd.DataFrame. + zipchunks : boolean, default=False + Set to True if merging post-processing files grouped into tar archives + as generated by the zipchunks approach. + """ # ------------------------------------------------------------------------- # MERGE POSTPRO ON NODE APPROACH INTO ONE DataFrame # ------------------------------------------------------------------------- lf = windIO.LogFile() path_pattern = os.path.join(P_RUN, 'logfiles', '*', '*.csv') + if zipchunks: + path_pattern = os.path.join(POST_DIR, 'loganalysis_chnk*.tar.xz') csv_fname = '%s_ErrorLogs.csv' % sim_id fcsv = os.path.join(POST_DIR, csv_fname) mdf = AppendDataFrames(tqdm=tqdm) @@ -489,6 +498,8 @@ def postpro_node_merge(tqdm=False): # ------------------------------------------------------------------------- path_pattern = os.path.join(P_RUN, 'res', '*', '*.csv') csv_fname = '%s_statistics.csv' % sim_id + if zipchunks: + path_pattern = os.path.join(POST_DIR, 'statsdel_chnk*.tar.xz') fcsv = os.path.join(POST_DIR, csv_fname) mdf = AppendDataFrames(tqdm=tqdm) # individual log file analysis does not have header, make sure to include @@ -636,7 +647,8 @@ if __name__ == '__main__': print('Start creating all the htc files and pbs_in files...') launch_dlcs_excel(sim_id, silent=False, zipchunks=opt.zipchunks, pbs_turb=opt.pbs_turb, walltime=opt.walltime, - postpro_node=opt.postpro_node, runmethod=RUNMETHOD) + postpro_node=opt.postpro_node, runmethod=RUNMETHOD, + dlcs_dir=os.path.join(P_SOURCE, 'htc', 'DLCs')) # post processing: check log files, calculate statistics if opt.check_logs or opt.stats or opt.fatigue or opt.envelopeblade \ or opt.envelopeturbine or opt.AEP: @@ -649,7 +661,7 @@ if __name__ == '__main__': envelopeturbine=opt.envelopeturbine, envelopeblade=opt.envelopeblade) if opt.postpro_node_merge: - postpro_node_merge() + postpro_node_merge(zipchunks=opt.zipchunks) if opt.dlcplot: plot_chans = {} plot_chans['$B1_{flap}$'] = ['setbeta-bladenr-1-flapnr-1']