prepost.dlctemplate: add postpro merge support for zipchunks,

optionally specify alternative dlcs location

wetb/prepost/dlctemplate.py

@@ -265,7 +265,8 @@ def variable_tag_func_mod1(master, case_id_short=False):
 
 def launch_dlcs_excel(sim_id, silent=False, verbose=False, pbs_turb=False,
                       runmethod=None, write_htc=True, zipchunks=False,
-                      walltime='04:00:00', postpro_node=False):
+                      walltime='04:00:00', postpro_node=False,
+                      dlcs_dir='htc/DLCs'):
     """
     Launch load cases defined in Excel files
     """
@@ -279,7 +280,6 @@ def launch_dlcs_excel(sim_id, silent=False, verbose=False, pbs_turb=False,
         pyenv = None
 
     # see if a htc/DLCs dir exists
-    dlcs_dir = os.path.join(P_SOURCE, 'htc', 'DLCs')
     # Load all DLC definitions and make some assumptions on tags that are not
     # defined
     if os.path.exists(dlcs_dir):
@@ -458,11 +458,14 @@ def post_launch(sim_id, statistics=True, rem_failed=True, check_logs=True,
     return df_stats, df_AEP, df_Leq
 
 
-def postpro_node_merge(tqdm=False):
+def postpro_node_merge(tqdm=False, zipchunks=False):
     """With postpro_node each individual case has a .csv file for the log file
     analysis and a .csv file for the statistics tables. Merge all these single
     files into one table/DataFrame.
 
+    When using the zipchunk approach, all log file analysis and statistics
+    are grouped into tar archives in the prepost-data directory.
+
     Parameters
     ----------
 
@@ -470,12 +473,18 @@ def postpro_node_merge(tqdm=False):
         Set to True for displaying a progress bar (provided by the tqdm module)
         when merging all csv files into a single table/pd.DataFrame.
 
+    zipchunks : boolean, default=False
+        Set to True if merging post-processing files grouped into tar archives
+        as generated by the zipchunks approach.
+
     """
     # -------------------------------------------------------------------------
     # MERGE POSTPRO ON NODE APPROACH INTO ONE DataFrame
     # -------------------------------------------------------------------------
     lf = windIO.LogFile()
     path_pattern = os.path.join(P_RUN, 'logfiles', '*', '*.csv')
+    if zipchunks:
+        path_pattern = os.path.join(POST_DIR, 'loganalysis_chnk*.tar.xz')
     csv_fname = '%s_ErrorLogs.csv' % sim_id
     fcsv = os.path.join(POST_DIR, csv_fname)
     mdf = AppendDataFrames(tqdm=tqdm)
@@ -489,6 +498,8 @@ def postpro_node_merge(tqdm=False):
     # -------------------------------------------------------------------------
     path_pattern = os.path.join(P_RUN, 'res', '*', '*.csv')
     csv_fname = '%s_statistics.csv' % sim_id
+    if zipchunks:
+        path_pattern = os.path.join(POST_DIR, 'statsdel_chnk*.tar.xz')
     fcsv = os.path.join(POST_DIR, csv_fname)
     mdf = AppendDataFrames(tqdm=tqdm)
     # individual log file analysis does not have header, make sure to include
@@ -636,7 +647,8 @@ if __name__ == '__main__':
         print('Start creating all the htc files and pbs_in files...')
         launch_dlcs_excel(sim_id, silent=False, zipchunks=opt.zipchunks,
                           pbs_turb=opt.pbs_turb, walltime=opt.walltime,
-                          postpro_node=opt.postpro_node, runmethod=RUNMETHOD)
+                          postpro_node=opt.postpro_node, runmethod=RUNMETHOD,
+                          dlcs_dir=os.path.join(P_SOURCE, 'htc', 'DLCs'))
     # post processing: check log files, calculate statistics
     if opt.check_logs or opt.stats or opt.fatigue or opt.envelopeblade \
         or opt.envelopeturbine or opt.AEP:
@@ -649,7 +661,7 @@ if __name__ == '__main__':
                     envelopeturbine=opt.envelopeturbine,
                     envelopeblade=opt.envelopeblade)
     if opt.postpro_node_merge:
-        postpro_node_merge()
+        postpro_node_merge(zipchunks=opt.zipchunks)
     if opt.dlcplot:
         plot_chans = {}
         plot_chans['$B1_{flap}$'] = ['setbeta-bladenr-1-flapnr-1']