v11: Ask GEOS-IT question first, look for MERRA-2

CHANGELOG.md

@@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- Update `remap_restarts.py` to first ask about remapping from GEOS-IT, then ask about MERRA-2. Add detection of MERRA-2 path as well
+
 ### Fixed
 
 - Edited gcmpost.script to move up the location of $SOURCE/plot/.quickplotrc to restore missing plots from landscape.list for some users.

pre/remap_restart/remap_analysis.py

@@ -17,7 +17,7 @@
 import fileinput
 import ruamel.yaml
 from remap_base import remap_base
-from remap_utils import get_label
+from remap_utils import get_label, MERRA2_RST_BASE
 import fnmatch
 
 class analysis(remap_base):
@@ -95,9 +95,9 @@ def remap(self):
      agrid_out = config['output']['shared']['agrid']
      flags = "-g5 -res " + self.get_grid_kind(agrid_out.upper()) + " -nlevs " + str(nlevel)
 
-     dyn2dyn = fnmatch.filter(local_fs, '*bkg??_eta_rst*') 
+     dyn2dyn = fnmatch.filter(local_fs, '*bkg??_eta_rst*')
      for f in dyn2dyn:
-        if "cbkg" not in f: 
+        if "cbkg" not in f:
           f_orig = f + ".orig"
           shutil.move(f,f_orig)
           cmd = bindir + '/dyn2dyn.x ' + flags + ' -o ' + f + ' ' + f_orig
@@ -106,15 +106,15 @@ def remap(self):
 
      for f in local_fs:
        fname = os.path.basename(f)
-       fname = fname + label 
+       fname = fname + label
        shutil.move(f, out_dir+'/'+fname)
     # write lcv file
-    #  (Info about "lcv" file provided by Ricardo Todling via email, 1 Sep 2023, 
+    #  (Info about "lcv" file provided by Ricardo Todling via email, 1 Sep 2023,
     #   paraphrased by Rolf Reichle:
     #   The lcv file is binary and inherits the nomenclature of the old GCM primary restarts [GEOS-4].
-    #   The file simply carries the date of the restarts, e.g., 20231201 210000. 
-    #   The file is required by the ADAS. 
-    #   LCV stands for Lagrangian Control Volume, which is what the grid coordinates of the model are 
+    #   The file simply carries the date of the restarts, e.g., 20231201 210000.
+    #   The file is required by the ADAS.
+    #   LCV stands for Lagrangian Control Volume, which is what the grid coordinates of the model are
     #   [on the cubed-sphere for GEOS-5].)
      lcv = config['output']['analysis']['lcv']
      if lcv :
@@ -174,15 +174,15 @@ def copy_merra2(self):
     dd_   = yyyymmddhh_[6:8]
     hh_   = yyyymmddhh_[8:10]
 
-    merra_2_rst_dir = '/archive/users/gmao_ops/MERRA2/gmao_ops/GEOSadas-5_12_4/'+expid +'/rs/Y'+yyyy_ +'/M'+mm_+'/'
+    merra_2_rst_dir = MERRA2_RST_BASE + expid +'/rs/Y'+yyyy_ +'/M'+mm_+'/'
     rst_dir = self.config['input']['shared']['rst_dir'] + '/'
     os.makedirs(rst_dir, exist_ok = True)
     print(' Stage MERRA-2 analysis files \n from \n    ' + merra_2_rst_dir + '\n to\n    '+ rst_dir +'\n')
 
     rst_time = datetime(year=int(yyyy_), month=int(mm_), day=int(dd_), hour = int(hh_))
 
     anafiles=[]
-    canafiles=[] 
+    canafiles=[]
     for h in [3,4,5,6,7,8,9]:
        delt = timedelta(hours = h-3)
        new_time = rst_time + delt
@@ -218,7 +218,7 @@ def copy_merra2(self):
        else:
           print('Warning: Cannot find '+f)
 
-       # satb files 
+       # satb files
        for ftype in ['satbias', 'satbang']:
           fname = expid + '.ana_'+ftype+'_rst.'+ymd+'_'+newhh+'z.txt'
           f = m2_rst_dir+'/'+fname

pre/remap_restart/remap_base.py

@@ -40,7 +40,7 @@ def remove_geosit(self):
 
   def copy_without_remap(self, restarts_in, compared_file_in, compared_file_out, suffix, catch=False):
 #
-#    Determine if remapping is needed for a group of restart files, or if the input restart files 
+#    Determine if remapping is needed for a group of restart files, or if the input restart files
 #    can just simply be copied to the output dir, based on the following dependency table:
 #
 #    restarts     | agrid/stretch | #levels | topo files | tile file | bcs version
@@ -96,21 +96,21 @@ def copy_without_remap(self, restarts_in, compared_file_in, compared_file_out, s
 
   def copy_geosit(self):
     if not self.config['input']['shared']['GEOS-IT']:
-        return 
-     
+        return
+
     expid = self.config['input']['shared']['expid']
     yyyymmddhh_ = str(self.config['input']['shared']['yyyymmddhh'])
     yyyy_ = yyyymmddhh_[0:4]
     mm_   = yyyymmddhh_[4:6]
     day_  = yyyymmddhh_[6:8]  # Extract the day from yyyymmddhh_
-          
-    time_suffix = '_21z' 
-    time_suffix_nc4 = '_2100z' 
-        
-    geos_it_rst_dir = '/discover/nobackup/projects/gmao/geos-it/dao_ops/archive/' + expid + '/rs/Y' + yyyy_ + '/M' + mm_ + '/'
+
+    time_suffix = '_21z'
+    time_suffix_nc4 = '_2100z'
+
+    geos_it_rst_dir = GEOSIT_RST_BASE + expid + '/rs/Y' + yyyy_ + '/M' + mm_ + '/'
     rst_dir = self.config['input']['shared']['rst_dir'] + '/'
     os.makedirs(rst_dir, exist_ok=True)
-             
+
     print('Stage GEOS-IT restarts \n from \n    ' + geos_it_rst_dir + '\n to\n    ' + rst_dir + '\n')
 
     # Only use the specific day from yyyymmddhh_

pre/remap_restart/remap_catchANDcn.py

@@ -146,6 +146,7 @@ def remap(self):
        CONSTRAINT = '"[cas|mil]"'
 
      account    = config['slurm_pbs']['account']
+
      # even if the (MERRA-2) input restarts are binary, the output restarts will always be nc4 (remap_bin2nc.py)
      suffix = '_rst.' + suffix
      out_rstfile = expid + os.path.basename(in_rstfile).split('_rst')[0].split('.')[-1]+suffix
@@ -241,7 +242,7 @@ def copy_merra2(self):
     hh_   = yyyymmddhh_[8:10]
 
     suffix = yyyymmddhh_[0:8]+'_'+ hh_ + 'z.bin'
-    merra_2_rst_dir = '/archive/users/gmao_ops/MERRA2/gmao_ops/GEOSadas-5_12_4/'+expid +'/rs/Y'+yyyy_ +'/M'+mm_+'/'
+    merra_2_rst_dir = MERRA2_RST_BASE + expid +'/rs/Y'+yyyy_ +'/M'+mm_+'/'
     rst_dir = self.config['input']['shared']['rst_dir'] + '/'
     os.makedirs(rst_dir, exist_ok = True)
     print(' Copy MERRA-2 catchment Restart \n from \n    ' + merra_2_rst_dir + '\n to\n    '+ rst_dir +'\n')

pre/remap_restart/remap_command_line.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 #
 # remap_restarts package:
-#   remap_command_line.py parses and converts the command-line arguments and converts the information 
+#   remap_command_line.py parses and converts the command-line arguments and converts the information
 #   into matching "Answers" for the questionary (remap_questions.py)
 #
 import os
@@ -64,7 +64,7 @@ def parse_args(program_description):
     p_command.add_argument('-ocnmdlout',   default='data',    help='Ocean model for new restarts',          choices=choices_omodel)
     p_command.add_argument('-in_stretch',  default=False,     help='Stretched CS params of input restarts', choices=choices_stretch)
     p_command.add_argument('-out_stretch', default=False,     help='Stretched CS params for new restarts',  choices=choices_stretch)
-    
+
     # Unlike remap_questions.py, command-line feature does not deduce Catch vs. CatchCN[40,51] for simplicity, thus requires input argument
     p_command.add_argument('-catch_model',default='catch',    help='Catchment[CN] model', choices=choices_catchmodel)
 
@@ -99,14 +99,14 @@ def get_answers_from_command_line(cml):
       init_merra2(answers)
    elif  cml.geosit:
       init_geosit(answers)
-   else:   
+   else:
       answers["input:shared:bc_base"]     =  cml.in_bc_base
       answers["input:shared:omodel"]      = cml.ocnmdlin
       answers["input:shared:bc_version"]  = cml.bcvin
       answers["input:surface:catch_model"]= cml.catch_model
       answers["input:shared:stretch"]     = cml.in_stretch
       answers["input:shared:rst_dir"]     = os.path.abspath(os.path.expanduser(cml.rst_dir))
-      fvcore_info(answers) 
+      fvcore_info(answers)
       ogrid                               = cml.oceanin
       if ogrid == "CS":
          ogrid = answers["input:shared:agrid"]
@@ -143,7 +143,7 @@ def get_answers_from_command_line(cml):
 
    answers["output:air:agcm_import_rst"] = not cml.noagcm_import_rst
 
-   if cml.zoom: 
+   if cml.zoom:
       answers["input:surface:zoom"]    = cml.zoom
    else:
       answers["input:surface:zoom"]    = get_zoom(answers)
@@ -161,7 +161,7 @@ def get_answers_from_command_line(cml):
    answers["slurm_pbs:account"]    = cml.account
    answers["slurm_pbs:qos"]        = cml.qos
    answers["slurm_pbs:partition"]  = cml.partition
-  
+
    return answers
 
 if __name__ == "__main__":
@@ -174,7 +174,7 @@ def get_answers_from_command_line(cml):
    with open("raw_command.yaml", "w") as f:
      yaml.dump(config, f)
 
-   config = get_config_from_answers(answers, config_tpl= True) 
+   config = get_config_from_answers(answers, config_tpl= True)
    with open("params_from_command.yaml", "w") as f:
      yaml.dump(config, f)
 

pre/remap_restart/remap_lake_landice_saltwater.py

@@ -17,7 +17,7 @@
 import ruamel.yaml
 import shlex
 from remap_base   import remap_base
-from remap_utils  import get_label, get_geomdir, get_zoom
+from remap_utils  import get_label, get_geomdir, get_zoom, MERRA2_RST_BASE
 from remap_bin2nc import bin2nc
 
 class lake_landice_saltwater(remap_base):
@@ -138,7 +138,7 @@ def remap(self):
      zoom = config['input']['surface']['zoom']
      if zoom is None :
         zoom = get_zoom(config)
- 
+
      log_name = out_dir+'/remap_lake_landice_saltwater_log'
      if os.path.exists(log_name):
         os.remove(log_name)
@@ -264,7 +264,7 @@ def copy_merra2(self):
     hh_   = yyyymmddhh_[8:10]
 
     suffix = yyyymmddhh_[0:8]+'_'+ hh_ + 'z.bin'
-    merra_2_rst_dir = '/archive/users/gmao_ops/MERRA2/gmao_ops/GEOSadas-5_12_4/'+expid +'/rs/Y'+yyyy_ +'/M'+mm_+'/'
+    merra_2_rst_dir = MERRA2_RST_BASE + expid +'/rs/Y'+yyyy_ +'/M'+mm_+'/'
     rst_dir = self.config['input']['shared']['rst_dir'] + '/'
     os.makedirs(rst_dir, exist_ok = True)
     print(' Copy MERRA-2 surface restarts \n from \n    ' + merra_2_rst_dir + '\n to\n    '+ rst_dir +'\n')