From e0d8ea7dce11d115ab8cd9c363b1e7adabfdec88 Mon Sep 17 00:00:00 2001
From: William Putman <William.M.Putman@nasa.gov>
Date: Wed, 31 Jan 2024 15:25:53 -0500
Subject: [PATCH 01/24] updated stretched cube input options for upper restarts

---
 post/stats.F90                          | 12 ++++++++--
 pre/remap_restart/bin2nc_merra2_fv.yaml |  8 +++----
 pre/remap_restart/remap_bin2nc.py       | 12 +++++-----
 pre/remap_restart/remap_catchANDcn.py   | 11 +++++----
 pre/remap_restart/remap_command_line.py |  2 +-
 pre/remap_restart/remap_questions.py    |  4 ++--
 pre/remap_restart/remap_upper.py        | 32 +++++++++++++++++++------
 7 files changed, 54 insertions(+), 27 deletions(-)

diff --git a/post/stats.F90 b/post/stats.F90
index 7658325a..581bfe7c 100644
--- a/post/stats.F90
+++ b/post/stats.F90
@@ -1046,8 +1046,16 @@ end subroutine init_levs
                     zlev(lev).eq.750 .or. zlev(lev).eq.600 .or. &
                     zlev(lev).eq.500 .or. zlev(lev).eq.400 .or. &
                     zlev(lev).eq.300 .or. zlev(lev).eq.200 .or. &
-                    zlev(lev).eq.100 .or. zlev(lev).eq.70  .or. &
-                    zlev(lev).eq.30  .or. zlev(lev).eq.10  ) then
+                    zlev(lev).eq.100 .or. zlev(lev).eq.90  .or. &
+                    zlev(lev).eq.80  .or. zlev(lev).eq.70  .or. &
+                    zlev(lev).eq.60  .or. zlev(lev).eq.50  .or. &
+                    zlev(lev).eq.40  .or. zlev(lev).eq.30  .or. &
+                    zlev(lev).eq.20  .or. zlev(lev).eq.10  .or. &
+                    zlev(lev).eq.9   .or. zlev(lev).eq.8   .or. &
+                    zlev(lev).eq.7   .or. zlev(lev).eq.6   .or. &
+                    zlev(lev).eq.5   .or. zlev(lev).eq.4   .or. &
+                    zlev(lev).eq.3   .or. zlev(lev).eq.2   .or. &
+                    zlev(lev).eq.1 ) then
                     write(6,1005) int(zlev(lev)),trim(regions(iregion)),&
                                   trim(fields_3d(n)%name),nymd,nhms,hour,&
                                   rms(iregion,lev,nfield,nt,1),&
diff --git a/pre/remap_restart/bin2nc_merra2_fv.yaml b/pre/remap_restart/bin2nc_merra2_fv.yaml
index 4756f29a..2d9ecb3b 100644
--- a/pre/remap_restart/bin2nc_merra2_fv.yaml
+++ b/pre/remap_restart/bin2nc_merra2_fv.yaml
@@ -3,12 +3,12 @@ variables:
     long_name: hybrid_sigma_pressure_a
     units: 'Pa'
     dimension:
-       - edges
+       - edge
   - short_name: BK
     long_name: hybrid_sigma_pressure_b
     units: '1'
     dimension:
-       - edges
+       - edge
   - short_name: U
     long_name: eastward_wind
     units: 'm s-1'
@@ -34,7 +34,7 @@ variables:
     long_name: air_pressure
     units: 'Pa'
     dimension:
-       - edges
+       - edge
        - lat
        - lon
   - short_name: PKZ
@@ -62,5 +62,5 @@ dimensions:
   lon: 180
   lat: 1080
   lev: 72
-  edges: 73
+  edge: 73
   time: 1
diff --git a/pre/remap_restart/remap_bin2nc.py b/pre/remap_restart/remap_bin2nc.py
index f7b94ed6..ec06d278 100755
--- a/pre/remap_restart/remap_bin2nc.py
+++ b/pre/remap_restart/remap_bin2nc.py
@@ -17,7 +17,7 @@ def writeCVars(coordVars,dimensions):
    for v in coordVars:
        if v=='time':
           coordVars[v][:]=0
-       elif (v=='lat') | (v=='lon') | (v=='lev') | (v=='edges'):
+       elif (v=='lat') | (v=='lon') | (v=='lev') | (v=='edge'):
           dsize=dimensions[v]
           coords=np.arange(1,dsize+1)
           coordVars[v][:]=coords
@@ -55,8 +55,8 @@ def writeVar(v, vars, dimensions, bintype, binf):
          for i in range(dimensions.get('lev')):
             rec = binf.read_reals(dtype=bintype)
             var[i,:,:]=rec
-      elif 'edges' in dims:
-         for i in range(dimensions.get('edges')):
+      elif 'edge' in dims:
+         for i in range(dimensions.get('edge')):
             rec = binf.read_reals(dtype=bintype)
             var[i,:,:]=rec
       else:
@@ -64,7 +64,7 @@ def writeVar(v, vars, dimensions, bintype, binf):
          var[:,:]=rec
    # is just lev
    else:
-      if 'edges' in dims:
+      if 'edge' in dims:
          rec = binf.read_reals(dtype=bintype)
          var[:]=rec
 
@@ -90,10 +90,10 @@ def defineDimVars(fid,dims):
           setattr(newVar,'positive','down')
           setattr(newVar,'formulaTerms','ap: ak b: bk ps: ps p0: p00')
           coordVars.update([(d,newVar)])
-       if d=="edges":
+       if d=="edge":
           newVar=fid.createVariable(d,'f8',d)
           setattr(newVar,'units','level')
-          setattr(newVar,'long_name','sigma_at_layer edges')
+          setattr(newVar,'long_name','sigma_at_layer edge')
           setattr(newVar,'standard_name','atmosphere_hybrid_sigma_pressure_coordinate')
           setattr(newVar,'coordinate','eta')
           setattr(newVar,'positive','down')
diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py
index c374f493..6aa1a7a0 100755
--- a/pre/remap_restart/remap_catchANDcn.py
+++ b/pre/remap_restart/remap_catchANDcn.py
@@ -104,11 +104,12 @@ def remap(self):
      else:
         NPE = 160
 
-     QOS = "#SBATCH --qos="+config['slurm']['qos']
-     TIME ="#SBATCH --time=1:00:00"
-     if NPE >= 160:
-       assert config['slurm']['qos'] != 'debug', "qos should be allnccs"
-       TIME = "#SBATCH --time=12:00:00"
+#    QOS = "#SBATCH --qos="+config['slurm']['qos']
+     TIME ="#SBATCH --time=3:00:00"
+#    if NPE >= 160:
+#      assert config['slurm']['qos'] != 'debug', "qos should be allnccs"
+#      TIME = "#SBATCH --time=12:00:00"
+     QOS = "#SBATCH --constraint=mil"
      PARTITION = "#SBATCH --partition=" + config['slurm']['partition']
     
      account    = config['slurm']['account']
diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py
index 2a7693ff..51d38d13 100755
--- a/pre/remap_restart/remap_command_line.py
+++ b/pre/remap_restart/remap_command_line.py
@@ -75,7 +75,7 @@ def parse_args(program_description):
     p_command.add_argument('-qos',        default="debug",    help="SLURM quality-of-service", choices=['debug', 'allnccs'])
     account = get_account()
     p_command.add_argument('-account',    default=account,    help="SLURM account")
-    p_command.add_argument('-partition',  default='compute',  help="SLURM partition")
+    p_command.add_argument('-partition',  default='scutest',  help="SLURM partition")
     p_command.add_argument('-rs',         default='3',        help="Flag indicating which restarts to regrid: 1 (upper air); 2 (surface); 3 (both)", choices=['1','2','3'])
 
     # Parse using parse_known_args so we can pass the rest to the remap scripts
diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py
index b3839005..d1280325 100755
--- a/pre/remap_restart/remap_questions.py
+++ b/pre/remap_restart/remap_questions.py
@@ -57,7 +57,7 @@ def echo_bcs(x,opt):
 
 def default_partition(x):
    if x['slurm:qos'] == 'debug':
-      x['slurm:partition'] = 'compute'
+      x['slurm:partition'] = 'scutest'
       return False
    return True
 
@@ -425,7 +425,7 @@ def ask_questions():
             "type": "text",
             "name": "slurm:partition",
             "message": "SLURM partition?",
-            "default": "compute",
+            "default": "scutest",
             "when": lambda x : default_partition(x),
         },
    ]
diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py
index 30ceacb7..e567f43a 100755
--- a/pre/remap_restart/remap_upper.py
+++ b/pre/remap_restart/remap_upper.py
@@ -143,12 +143,13 @@ def remap(self):
      elif (imout>=2880):
        NPE = 5400; nwrit = 6
 
-     QOS = "#SBATCH --qos="+config['slurm']['qos']
-     TIME ="#SBATCH --time=1:00:00"
-     if NPE > 532: 
-       assert config['slurm']['qos'] != 'debug', "qos should be allnccs"
-       TIME = "#SBATCH --time=12:00:00"
+#    QOS = "#SBATCH --qos="+config['slurm']['qos']
+     TIME ="#SBATCH --time=3:00:00"
+#    if NPE > 532: 
+#      assert config['slurm']['qos'] != 'debug', "qos should be allnccs"
+#      TIME = "#SBATCH --time=12:00:00"
 
+     QOS = "#SBATCH --constraint=mil"
      PARTITION = "#SBATCH --partition=" + config['slurm']['partition']
 
      log_name = out_dir+'/remap_upper_log'
@@ -158,7 +159,7 @@ def remap(self):
      # to interp_restarts.x. Per the code we use:
      #  -stretched_grid target_lon target_lat stretch_fac
      # If we are not running stretched grid, we should pass in a blank string
-     stretch = config['output']['shared']['stretch']
+     stretch = config['input']['shared']['stretch']
      stretch_str = ""
      if stretch:
         if stretch == 'SG001':
@@ -174,7 +175,24 @@ def remap(self):
 
         # note "reversed" order of args (relative to order in definition of STRETCH_GRID)  
 
-        stretch_str = "-stretched_grid " + str(target_lon) + " " + str(target_lat) + " " + str(stretch_fac)
+        stretch_str = "-stretched_grid_in " + str(target_lon) + " " + str(target_lat) + " " + str(stretch_fac)
+
+     stretch = config['output']['shared']['stretch']
+     if stretch:
+        if stretch == 'SG001':
+          stretch_fac = STRETCH_GRID['SG001'][0]
+          target_lat  = STRETCH_GRID['SG001'][1]
+          target_lon  = STRETCH_GRID['SG001'][2]
+        elif stretch == 'SG002':
+          stretch_fac = STRETCH_GRID['SG002'][0]
+          target_lat  = STRETCH_GRID['SG002'][1]
+          target_lon  = STRETCH_GRID['SG002'][2]
+        else:
+          exit("This stretched grid option is not supported " + str(stretch))
+     
+        # note "reversed" order of args (relative to order in definition of STRETCH_GRID)  
+     
+        stretch_str = stretch_str+" -stretched_grid_out " + str(target_lon) + " " + str(target_lat) + " " + str(stretch_fac)
 
      # Now, let's create the input.nml file
      # We need to create a namelist for the upper air remapping

From 176e0de61f93d28954d99509ed9626d28b615080 Mon Sep 17 00:00:00 2001
From: Weiyuan Jiang <weiyuan.jiang@nasa.gov>
Date: Wed, 21 Feb 2024 14:47:34 -0500
Subject: [PATCH 02/24] change log

---
 CHANGELOG.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fdd471b1..df3cf649 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added function to remap from stretched restarts to stretched restarts
+
 ### Changed
 
 ### Fixed

From 2815bfda76458afb1cc5389ef3ad3fa2317aa4e2 Mon Sep 17 00:00:00 2001
From: Weiyuan Jiang <52509753+weiyuan-jiang@users.noreply.github.com>
Date: Mon, 26 Feb 2024 11:53:16 -0500
Subject: [PATCH 03/24] Update remap_command_line.py

correct typo
---
 pre/remap_restart/remap_command_line.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py
index 2f93d8e9..67c5c187 100755
--- a/pre/remap_restart/remap_command_line.py
+++ b/pre/remap_restart/remap_command_line.py
@@ -79,7 +79,7 @@ def parse_args(program_description):
     p_command.add_argument('-qos',        default="debug",    help="slurm_pbs quality-of-service", choices=['debug', 'allnccs', 'normal'])
     account = get_account()
     p_command.add_argument('-account',    default=account,    help="slurm_pbs account")
-    if (BUILDT_ON_SLES15):
+    if (BUILT_ON_SLES15):
       p_command.add_argument('-partition',  default='scutest',help="slurm_pbs partition")
     else:
       p_command.add_argument('-partition',  default='',       help="slurm_pbs partition")

From 23ecea54f02e482c058c1cb7de83fe298ddf0153 Mon Sep 17 00:00:00 2001
From: William Putman <William.M.Putman@nasa.gov>
Date: Tue, 5 Mar 2024 12:48:17 -0500
Subject: [PATCH 04/24] major updates for upper_air restart regrid/remap to fix
 vector regrid and remapping options

---
 pre/remap_restart/remap_command_line.py |  5 +----
 pre/remap_restart/remap_params.tpl      |  2 +-
 pre/remap_restart/remap_upper.py        | 22 +++++++++++++---------
 3 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py
index 2f93d8e9..9b59632b 100755
--- a/pre/remap_restart/remap_command_line.py
+++ b/pre/remap_restart/remap_command_line.py
@@ -79,10 +79,7 @@ def parse_args(program_description):
     p_command.add_argument('-qos',        default="debug",    help="slurm_pbs quality-of-service", choices=['debug', 'allnccs', 'normal'])
     account = get_account()
     p_command.add_argument('-account',    default=account,    help="slurm_pbs account")
-    if (BUILDT_ON_SLES15):
-      p_command.add_argument('-partition',  default='scutest',help="slurm_pbs partition")
-    else:
-      p_command.add_argument('-partition',  default='',       help="slurm_pbs partition")
+    p_command.add_argument('-partition',  default='',         help="slurm_pbs partition")
     p_command.add_argument('-rs',         default='3',        help="Flag indicating which restarts to regrid: 1 (upper air); 2 (surface); 3 (both)", choices=['1','2','3'])
 
     # Parse using parse_known_args so we can pass the rest to the remap scripts
diff --git a/pre/remap_restart/remap_params.tpl b/pre/remap_restart/remap_params.tpl
index 1e7f6552..8d670bba 100644
--- a/pre/remap_restart/remap_params.tpl
+++ b/pre/remap_restart/remap_params.tpl
@@ -7,7 +7,7 @@
 input:
   air:
     drymass: 1
-    hydrostatic: 0
+    hydrostatic: true
   shared:
     MERRA-2: false
     stretch: false
diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py
index 4a289646..b134ba83 100755
--- a/pre/remap_restart/remap_upper.py
+++ b/pre/remap_restart/remap_upper.py
@@ -235,9 +235,6 @@ def remap(self):
 cd {out_dir}/upper_data
 /bin/touch input.nml
 
-# The MERRA fvcore_internal_restarts don't include W or DZ, but we can add them by setting
-# HYDROSTATIC = 0 which means HYDROSTATIC = FALSE
-
 if ($?I_MPI_ROOT) then
   # intel scaling suggestions
   #--------------------------
@@ -272,7 +269,14 @@ def remap(self):
     set ioflag = ""
 endif
 
-set drymassFLG = {drymassFLG}
+set hydrostaticIN = {hydrostatic}
+if ( $hydrostaticIN == 'True' ) then
+    set hydrostaticflag = "-in_hydrostatic T"
+else
+    set hydrostaticflag = "-in_hydrostatic F"
+endif
+
+set drymassFLG = {drymass}
 if ($drymassFLG) then
     set dmflag = ""
 else
@@ -280,20 +284,20 @@ def remap(self):
 endif
 
 {Bin}/esma_mpirun -np {NPE} $interp_restartsX -im {imout} -lm {nlevel} \\
-   -do_hydro {hydrostatic} $ioflag $dmflag -nwriter {nwrit} {stretch_str}
+   $hydrostaticflag {stretch_str} $dmflag -nwriter {nwrit} $ioflag 
 
 """
      account = config['slurm_pbs']['account']
-     drymassFLG  = config['input']['air']['drymass']
+     drymass     = config['input']['air']['drymass']
      hydrostatic = config['input']['air']['hydrostatic']
      nlevel = config['output']['air']['nlevel']
      log_name = out_dir+'/remap_upper_log'
      job_name = 'remap_upper'
      remap_upper_script = remap_template.format(Bin=bindir, account = account, \
-             out_dir = out_dir, log_name = log_name, job_name= job_name, drymassFLG = drymassFLG, \
+             out_dir = out_dir, log_name = log_name, job_name= job_name, drymass = drymass, \
              imout = imout, nwrit = nwrit, NPE = NPE, NNODE = NNODE, \
-             QOS = QOS, TIME = TIME, CONSTRAINT = CONSTRAINT, PARTITION = PARTITION, nlevel = nlevel, hydrostatic = hydrostatic,
-             stretch_str = stretch_str)
+             QOS = QOS, TIME = TIME, CONSTRAINT = CONSTRAINT, PARTITION = PARTITION, nlevel = nlevel, \
+             hydrostatic = hydrostatic, stretch_str = stretch_str)
 
      script_name = './remap_upper.j'
 

From 7cf1841aedb01da1250e5bd26f6cdbee0959395e Mon Sep 17 00:00:00 2001
From: Weiyuan Jiang <weiyuan.jiang@nasa.gov>
Date: Sat, 9 Mar 2024 10:07:58 -0500
Subject: [PATCH 05/24] added hydrostatic question and blank qos choice

---
 CHANGELOG.md                            |  2 ++
 pre/remap_restart/remap_catchANDcn.py   | 12 +++++++++---
 pre/remap_restart/remap_command_line.py |  4 +++-
 pre/remap_restart/remap_params.tpl      |  2 +-
 pre/remap_restart/remap_questions.py    |  8 ++++++++
 pre/remap_restart/remap_upper.py        |  7 ++++++-
 pre/remap_restart/remap_utils.py        | 19 +++++++++++++------
 7 files changed, 42 insertions(+), 12 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index cdf279ee..19e4833e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added choice 'blank' for qos in remap_restart python pacakage
+- Added hydrostatic question in remap_restart python pacakage
 - Added function to remap from stretched restarts to stretched restarts
 
 ### Changed
diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py
index 004a5d1e..cf95f083 100755
--- a/pre/remap_restart/remap_catchANDcn.py
+++ b/pre/remap_restart/remap_catchANDcn.py
@@ -121,9 +121,11 @@ def remap(self):
         NPE = 160
 
      PARTITION =''
-     QOS  = config['slurm_pbs']['qos']
+     QOS       =''
+
+     qos  = config['slurm_pbs']['qos']
      TIME  = "1:00:00"
-     if QOS != "debug": TIME="3:00:00"
+     if qos != "debug": TIME="3:00:00"
 
      NNODE = ''
      job = ''
@@ -131,6 +133,8 @@ def remap(self):
        job = "PBS"
        CONSTRAINT = 'cas_ait'
        NNODE = (NPE-1)//40 + 1
+       if (qos != ''):
+         QOS = "#PBS  -q "+qos
      else:
        job = "SLURM"
        partition = config['slurm_pbs']['partition']
@@ -140,6 +144,8 @@ def remap(self):
        CONSTRAINT = '"[cas|sky]"'
        if BUILT_ON_SLES15:
          CONSTRAINT = 'mil'
+       if (qos != ''):
+         QOS = "#SBATCH  --qos="+qos
 
      account    = config['slurm_pbs']['account']
 
@@ -363,7 +369,7 @@ def has_catch_rst(text):
             "type": "text",
             "name": "slurm_pbs:qos",
             "message": message_qos,
-            "default": "debug",
+            "default": "",
         },
 
         {
diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py
index 67c5c187..46daf386 100755
--- a/pre/remap_restart/remap_command_line.py
+++ b/pre/remap_restart/remap_command_line.py
@@ -67,6 +67,7 @@ def parse_args(program_description):
     # Unlike remap_questions.py, command-line feature does not deduce Catch vs. CatchCN[40,45] for simplicity, thus requires input argument
     p_command.add_argument('-catch_model',default='catch',    help='Catchment[CN] model', choices=choices_catchmodel)
 
+    p_command.add_argument('-nonhydrostatic', action='store_true',     help=" non hydrostatic upper air")
     p_command.add_argument('-nobkg', action='store_true',     help="Do not remap bkg files")
     p_command.add_argument('-nolcv', action='store_true',     help="Do not write lcv file")
     p_command.add_argument('-np',    action='store_true',     help="No prompt. Overwrite config files without prompting questions")
@@ -76,7 +77,7 @@ def parse_args(program_description):
     p_command.add_argument('-out_bc_base',default="",         help="Boundary conditions base dir (w/o bc_version and resolution info) for new restarts")
     p_command.add_argument('-zoom',                           help= "Zoom parameter (search radius) for input surface restarts")
 
-    p_command.add_argument('-qos',        default="debug",    help="slurm_pbs quality-of-service", choices=['debug', 'allnccs', 'normal'])
+    p_command.add_argument('-qos',        default="",    help="slurm_pbs quality-of-service", choices=['', 'debug', 'allnccs', 'normal'])
     account = get_account()
     p_command.add_argument('-account',    default=account,    help="slurm_pbs account")
     if (BUILT_ON_SLES15):
@@ -127,6 +128,7 @@ def get_answers_from_command_line(cml):
    answers["input:shared:stretch"]     = cml.in_stretch
    answers["output:analysis:bkg"]      = not cml.nobkg
    answers["output:analysis:lcv"]      = not cml.nolcv
+   answers["input:air:hydrostatic"]    = not cml.nonhydrostatic
    if cml.rs == '1':
      answers["output:air:remap"]            = True
      answers["output:surface:remap_water"]  = False
diff --git a/pre/remap_restart/remap_params.tpl b/pre/remap_restart/remap_params.tpl
index 8d670bba..4049eed8 100644
--- a/pre/remap_restart/remap_params.tpl
+++ b/pre/remap_restart/remap_params.tpl
@@ -62,5 +62,5 @@ output:
     lcv: false
 slurm_pbs:
   account:
-  qos:
+  qos: ''
   partition: ''
diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py
index 05ec4d4b..b58f59db 100755
--- a/pre/remap_restart/remap_questions.py
+++ b/pre/remap_restart/remap_questions.py
@@ -92,6 +92,13 @@ def ask_questions():
             "validate": lambda text: validate_merra2_time(text) ,
             "when": lambda x: x['input:shared:MERRA-2'],
         },
+        {
+            "type": "confirm",
+            "name": "input:air:hydrostatic",
+            "message": "Is the upper air input hydrostatic?\n",
+            "default": True,
+            "when": lambda x: not x['input:shared:MERRA-2'],
+        },
         {
             "type": "path",
             "name": "output:shared:out_dir",
@@ -428,6 +435,7 @@ def ask_questions():
    answers["output:surface:remap_water"] = answers["output:surface:remap"]
    answers["output:surface:remap_catch"] = answers["output:surface:remap"]
    del answers["output:surface:remap"]
+   if answers["input:shared:MERRA-2"] : answers["input:air:hydrostatic"] = True
  
    return answers
 
diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py
index b134ba83..c699e9eb 100755
--- a/pre/remap_restart/remap_upper.py
+++ b/pre/remap_restart/remap_upper.py
@@ -153,7 +153,8 @@ def remap(self):
        NPE = 5400; nwrit = 6
 
      PARTITION =''
-     QOS  = config['slurm_pbs']['qos']
+     QOS       =''
+     qos       = config['slurm_pbs']['qos']
      TIME = "1:00:00"
      if NPE > 532: 
         assert config['slurm_pbs']['qos'] != 'debug', "qos should be 'allnccs' for NCCS or 'normal' for NAS"
@@ -164,11 +165,15 @@ def remap(self):
        CONSTRAINT = 'cas_ait'
        NNODE = (NPE-1)//40 + 1
        job='PBS'
+       if (qos != ''):
+         QOS = "#PBS  -q "+qos
      else:
        job='SLURM'
        partition = config['slurm_pbs']['partition']
        if (partition != ''):
          PARTITION = "#SBATCH --partition=" + partition
+       if (qos != ''):
+         QOS = "#SBATCH  --qos="+qos
 
        CONSTRAINT = '"[cas|sky]"'
        if BUILT_ON_SLES15:
diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py
index 345a18bd..4ec96727 100755
--- a/pre/remap_restart/remap_utils.py
+++ b/pre/remap_restart/remap_utils.py
@@ -116,7 +116,10 @@
 
 message_ogrid_in   = "Select data ocean grid/resolution of input restarts:\n"
 
-message_qos        = "SLURM or PBS quality-of-service (qos)?  (If resolution is c1440 or higher, enter 'allnccs' for NCCS or 'normal' for NAS.)\n"
+message_qos        = """SLURM or PBS quality-of-service (qos)?
+    (Use default 'debug' to get resource faster; or
+     Enter 'allnccs' for NCCS or 'normal' for NAS if resolution is c1440 or higher;
+     or leave it blank)\n"""
 
 message_account    = "Select/enter SLURM or PBS account:\n",
 
@@ -128,20 +131,20 @@
 #SBATCH --ntasks={NPE}
 #SBATCH --job-name={job_name}
 #SBATCH --output={log_name}
-#SBATCH --qos={QOS}
 #SBATCH --time={TIME}
 #SBATCH --constraint={CONSTRAINT}
 {PARTITION}
+{QOS}
 """,
 "PBS": """#!/bin/csh -f
 #PBS -l walltime={TIME}
 #PBS -l select={NNODE}:ncpus=40:mpiprocs=40:model={CONSTRAINT}
 #PBS -N {job_name}
-#PBS -q {QOS}
 #PBS -W group_list={account}
 #PBS -o {log_name}
 #PBS -j oe
 {PARTITION}
+{QOS}
 """
 }
 
@@ -416,7 +419,8 @@ def get_command_line_from_answers(answers):
    
    nobkg  = '' if answers["output:analysis:bkg"] else " -nobkg "
    nolcv  = '' if answers["output:analysis:lcv"] else " -nolcv "
-  
+   nonhydrostatic = '' if answers["input:air:hydrostatic"] else " -nonhydrostatic " 
+ 
    label  = ' -lbl ' if answers["output:shared:label"] else ""
     
    in_bc_base  = ' -in_bc_base '  + answers.get("input:shared:bc_base")
@@ -447,7 +451,9 @@ def get_command_line_from_answers(answers):
    noagcm_import_rst  = '' if answers["output:air:agcm_import_rst"] else " -noagcm_import_rst "
 
    account = " -account " + answers["slurm_pbs:account"]
-   qos     = " -qos  " + answers["slurm_pbs:qos"]
+   qos = ''
+   if answers["slurm_pbs:qos"] != '':
+     qos     = " -qos  " + answers["slurm_pbs:qos"]
    partition = ''
    if answers["slurm_pbs:partition"] != '':
       partition  = " -partition  " + answers["slurm_pbs:partition"]
@@ -475,6 +481,7 @@ def get_command_line_from_answers(answers):
                                           wemout + \
                                           label + \
                                           nobkg + \
+                                          nonhydrostatic + \
                                           noagcm_import_rst + \
                                           nolcv + \
                                           out_rs + \
@@ -616,5 +623,5 @@ def remove_ogrid_comment(x, opt):
   return False
 
 if __name__ == '__main__' :
-   config = yaml_to_config('c24Toc12.yaml')
+   config = yaml_to_config('remap_params.tpl')
    print_config(config)

From d988c67b8be8d0389bb5558fbf1664a373a4f721 Mon Sep 17 00:00:00 2001
From: Weiyuan Jiang <weiyuan.jiang@nasa.gov>
Date: Mon, 1 Apr 2024 12:11:22 -0400
Subject: [PATCH 06/24] more clear message fo hydrostatic

---
 pre/remap_restart/remap_questions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py
index b58f59db..9c1391a6 100755
--- a/pre/remap_restart/remap_questions.py
+++ b/pre/remap_restart/remap_questions.py
@@ -95,7 +95,7 @@ def ask_questions():
         {
             "type": "confirm",
             "name": "input:air:hydrostatic",
-            "message": "Is the upper air input hydrostatic?\n",
+            "message": "Is the upper air input hydrostatic? (If you are not sure, don't change the default 'True')\n",
             "default": True,
             "when": lambda x: not x['input:shared:MERRA-2'],
         },

From eb8625e772a07b6ebf7ecdb1ff51ceb27aaef85d Mon Sep 17 00:00:00 2001
From: William Putman <William.M.Putman@nasa.gov>
Date: Fri, 5 Apr 2024 11:04:15 -0400
Subject: [PATCH 07/24] cleanup in/out grids and print values

---
 .../regrid_forcing_esmf.F90                   | 27 ++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/pre/NSIDC-OSTIA_SST-ICE_blend/regrid_forcing_esmf.F90 b/pre/NSIDC-OSTIA_SST-ICE_blend/regrid_forcing_esmf.F90
index 6b6ae02b..c42716a8 100644
--- a/pre/NSIDC-OSTIA_SST-ICE_blend/regrid_forcing_esmf.F90
+++ b/pre/NSIDC-OSTIA_SST-ICE_blend/regrid_forcing_esmf.F90
@@ -31,6 +31,10 @@ module GenESMFGridCompMod
      integer                   :: PJM
      integer                   :: GPIM
      integer                   :: GPJM
+     integer                   :: OIM
+     integer                   :: OJM
+     integer                   :: GOIM
+     integer                   :: GOJM
      type(ESMF_Time)           :: start_time
      type(ESMF_Time)           :: end_time
      logical                   :: select_time
@@ -277,6 +281,15 @@ subroutine Initialize ( GC, IMPORT, EXPORT, CLOCK, RC )
     PrivateState%ogrid=ogrid
     PrivateState%pgrid=pgrid
 
+! Query Ogrid to save IM and JM
+    call MAPL_GridGet(ogrid, localCellCountPerDim=COUNTS, &
+         globalCellCountPerDim=dims, RC=STATUS)
+    VERIFY_(STATUS)            
+    PrivateState%oim=counts(1) 
+    PrivateState%ojm=counts(2) 
+    PrivateState%goim=dims(1)  
+    PrivateState%gojm=dims(2)  
+
 ! Query Pgrid to save IM and JM
     call MAPL_GridGet(pgrid, localCellCountPerDim=COUNTS, &
          globalCellCountPerDim=dims, RC=STATUS)
@@ -346,6 +359,8 @@ subroutine RUN ( GC, IMPORT, EXPORT, CLOCK, RC )
 
   integer :: IM, JM
   integer :: IM_WORLD, JM_WORLD
+  integer :: OIM, OJM
+  integer :: OIM_WORLD, OJM_WORLD
   type(ESMF_Grid) :: ogrid, pgrid
   type(ESMF_VM) :: vm
   type(ESMF_Time) :: currentTime, dateN
@@ -360,6 +375,7 @@ subroutine RUN ( GC, IMPORT, EXPORT, CLOCK, RC )
   type (MAPL_MetaComp), pointer          :: MAPL 
   logical :: amIRoot,dowrite
   type(ESMF_Time) :: start_interval, end_interval
+  real    :: gmax, gmin
 
 !=============================================================================
 
@@ -405,6 +421,11 @@ subroutine RUN ( GC, IMPORT, EXPORT, CLOCK, RC )
 
    UNIT_W = getfile(filename)
 
+   OIM = PrivateState%oim
+   OJM = PrivateState%ojm
+   OIM_WORLD = PrivateState%goim
+   OJM_WORLD = PrivateState%gojm
+
    IM = PrivateState%pim
    JM = PrivateState%pjm
    IM_WORLD = PrivateState%gpim
@@ -484,13 +505,17 @@ subroutine RUN ( GC, IMPORT, EXPORT, CLOCK, RC )
    !   read(unit_r) odata
       call MAPL_VarRead(unit_r, grid=ogrid, a=odata, rc=status)
       VERIFY_(STATUS)
-      ! transform data from ocean (tripolar or Reynolds) to mit-cubed
+      ! Check global max/min
+      call MAPL_MaxMin('In data: ', odata)
+      ! transform data from ocean (tripolar or Reynolds) to cubed
       call privateState%regridder%regrid(odata,pdata,rc=status)
       VERIFY_(status) 
       if (privateState%fix_fraction) then
          where(pdata > 1.0) pdata = 1.0
          where(pdata < 0.0) pdata = 0.0
       end if
+      ! Check global max/min
+      call MAPL_MaxMin('Out data: ', pdata)
    !   write(unit_w) pdata
       if (doWrite) then
          call MAPL_VarWrite(unit_w, grid=pgrid, a=pdata, rc=status)

From b4a374efe5776bcf6e6f65a838c06a0c24491348 Mon Sep 17 00:00:00 2001
From: William Putman <William.M.Putman@nasa.gov>
Date: Wed, 10 Apr 2024 18:04:34 -0400
Subject: [PATCH 08/24] needed ENV variables for OpenMPI and IntelMPI on Milan
 nodes

---
 pre/remap_restart/remap_upper.py | 67 ++++++++++++++++++++++++++------
 pre/remap_restart/remap_utils.py |  2 -
 2 files changed, 56 insertions(+), 13 deletions(-)

diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py
index b134ba83..2f6c9326 100755
--- a/pre/remap_restart/remap_upper.py
+++ b/pre/remap_restart/remap_upper.py
@@ -235,18 +235,63 @@ def remap(self):
 cd {out_dir}/upper_data
 /bin/touch input.nml
 
-if ($?I_MPI_ROOT) then
-  # intel scaling suggestions
-  #--------------------------
-  setenv I_MPI_ADJUST_ALLREDUCE 12
-  setenv I_MPI_ADJUST_GATHERV 3
-
-  setenv I_MPI_SHM_HEAP_VSIZE 512
-  setenv PSM2_MEMORY large
-  setenv I_MPI_EXTRA_FILESYSTEM 1
-  setenv I_MPI_EXTRA_FILESYSTEM_FORCE gpfs
-  setenv ROMIO_FSTYPE_FORCE "gpfs:"
+set mpi_type = "openmpi"
+if ($?I_MPI_ROOT ) then 
+  set mpi_type = "intel"
 endif
+
+
+if ($mpi_type =~ "openmpi") then
+setenv OMPI_MCA_shmem_mmap_enable_nfs_warning 0
+# pre-connect MPI procs on mpi_init
+setenv OMPI_MCA_mpi_preconnect_all 1
+# options fo bcast: 0|ignore, 1|basic_linear, 2|chain, 3|pipeline, 4|split_binary_tree, 5|binary_tree, 6|binomial, 7|knomial, 8|scatter_allgather, 9|scatter_allgather_ring
+setenv OMPI_MCA_coll_tuned_bcast_algorithm 7
+# options for scatter: 0|ignore, 1|basic_linear, 2|binomial, 3|linear_nb
+setenv OMPI_MCA_coll_tuned_scatter_algorithm 2
+# options for reduce_scatter: 0|ignore, 1|non-overlapping, 2|recursive_halving, 3|ring, 4|butterfly
+setenv OMPI_MCA_coll_tuned_reduce_scatter_algorithm 3
+# options for allreduce: 0|ignore, 1|basic_linear, 2|nonoverlapping, 3|recursive_doubling, 4|ring, 5|segmented_ring, 6|rabenseifner
+setenv OMPI_MCA_coll_tuned_allreduce_algorithm 3
+# options for allgather: 0|ignore, 1|linear, 2|bruck, 3|recursive_doubling, 4|ring, 5|neighbor, 6|two_proc, 7|sparbit
+setenv OMPI_MCA_coll_tuned_allgather_algorithm 4
+# options for allgatherv: 0|ignore, 1|default, 2|bruck, 3|ring, 4|neighbor, 5|two_proc, 6|sparbit
+setenv OMPI_MCA_coll_tuned_allgatherv_algorithm 3
+# options for gather: 0 ignore, 1 basic linear, 2 binomial, 3 linear with synchronization
+setenv OMPI_MCA_coll_tuned_gather_algorithm 1
+# options for barrier: 0|ignore, 1|linear, 2|double_ring, 3|recursive_doubling, 4|bruck, 5|two_proc, 6|tree
+setenv OMPI_MCA_coll_tuned_barrier_algorithm 0
+# required for a tuned flag to be effective
+setenv OMPI_MCA_coll_tuned_use_dynamic_rules 1
+# disable file locks
+setenv OMPI_MCA_sharedfp "^lockedfile,individual"
+## HDF5: disable slow locks (promise not to open half-written files)
+#setenv HDF5_USE_FILE_LOCKING FALSE
+else
+setenv I_MPI_FABRICS ofi
+setenv I_MPI_OFI_PROVIDER psm3
+setenv I_MPI_ADJUST_SCATTER 2
+setenv I_MPI_ADJUST_SCATTERV 2
+setenv I_MPI_ADJUST_GATHER 2
+setenv I_MPI_ADJUST_GATHERV 3
+setenv I_MPI_ADJUST_ALLGATHER 3
+setenv I_MPI_ADJUST_ALLGATHERV 3
+setenv I_MPI_ADJUST_ALLREDUCE 12
+setenv I_MPI_ADJUST_REDUCE 10
+setenv I_MPI_ADJUST_BCAST 11
+setenv I_MPI_ADJUST_REDUCE_SCATTER 4
+setenv I_MPI_ADJUST_BARRIER 9
+#setenv I_MPI_SHM_HEAP_VSIZE 512
+#setenv I_MPI_EXTRA_FILESYSTEM 1
+#setenv I_MPI_EXTRA_FILESYSTEM_FORCE "gpfs"
+#setenv ROMIO_FSTYPE_FORCE "gpfs:"
+#setenv I_MPI_TUNING_MODE auto
+#setenv I_MPI_TUNING_BIN_DUMP tuning-results.dat
+#setenv I_MPI_DEBUG 6
+#setenv MPS_STAT_LEVEL 4
+env | grep 'I_MPI\|FI_'
+endif
+
 set infiles = ()
 set outfils = ()
 foreach infile ( *_restart_in )
diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py
index 345a18bd..69c31f0c 100755
--- a/pre/remap_restart/remap_utils.py
+++ b/pre/remap_restart/remap_utils.py
@@ -128,7 +128,6 @@
 #SBATCH --ntasks={NPE}
 #SBATCH --job-name={job_name}
 #SBATCH --output={log_name}
-#SBATCH --qos={QOS}
 #SBATCH --time={TIME}
 #SBATCH --constraint={CONSTRAINT}
 {PARTITION}
@@ -137,7 +136,6 @@
 #PBS -l walltime={TIME}
 #PBS -l select={NNODE}:ncpus=40:mpiprocs=40:model={CONSTRAINT}
 #PBS -N {job_name}
-#PBS -q {QOS}
 #PBS -W group_list={account}
 #PBS -o {log_name}
 #PBS -j oe

From 83389cf35128f421c8ac48f438c3c31f32075724 Mon Sep 17 00:00:00 2001
From: Matthew Thompson <matthew.thompson@nasa.gov>
Date: Thu, 12 Sep 2024 11:47:00 -0400
Subject: [PATCH 09/24] v12: Fix for SLES15 remapping

---
 CHANGELOG.md                            |  2 ++
 pre/remap_restart/remap_command_line.py | 19 ++++++++-----------
 pre/remap_restart/remap_questions.py    | 23 ++++++++++-------------
 3 files changed, 20 insertions(+), 24 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4e68664a..aaed5a39 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- Fixed issue with SLES15 in remapping
+
 ### Removed
 
 ### Deprecated
diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py
index 46daf386..574b4ca4 100755
--- a/pre/remap_restart/remap_command_line.py
+++ b/pre/remap_restart/remap_command_line.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 #
 # remap_restarts package:
-#   remap_command_line.py parses and converts the command-line arguments and converts the information 
+#   remap_command_line.py parses and converts the command-line arguments and converts the information
 #   into matching "Answers" for the questionary (remap_questions.py)
 #
 import os
@@ -63,7 +63,7 @@ def parse_args(program_description):
     p_command.add_argument('-ocnmdlout',   default='data',    help='Ocean model for new restarts',          choices=choices_omodel)
     p_command.add_argument('-in_stretch',  default=False,     help='Stretched CS params of input restarts', choices=choices_stretch)
     p_command.add_argument('-out_stretch', default=False,     help='Stretched CS params for new restarts',  choices=choices_stretch)
-    
+
     # Unlike remap_questions.py, command-line feature does not deduce Catch vs. CatchCN[40,45] for simplicity, thus requires input argument
     p_command.add_argument('-catch_model',default='catch',    help='Catchment[CN] model', choices=choices_catchmodel)
 
@@ -80,10 +80,7 @@ def parse_args(program_description):
     p_command.add_argument('-qos',        default="",    help="slurm_pbs quality-of-service", choices=['', 'debug', 'allnccs', 'normal'])
     account = get_account()
     p_command.add_argument('-account',    default=account,    help="slurm_pbs account")
-    if (BUILT_ON_SLES15):
-      p_command.add_argument('-partition',  default='scutest',help="slurm_pbs partition")
-    else:
-      p_command.add_argument('-partition',  default='',       help="slurm_pbs partition")
+    p_command.add_argument('-partition',  default='',       help="slurm_pbs partition")
     p_command.add_argument('-rs',         default='3',        help="Flag indicating which restarts to regrid: 1 (upper air); 2 (surface); 3 (both)", choices=['1','2','3'])
 
     # Parse using parse_known_args so we can pass the rest to the remap scripts
@@ -100,11 +97,11 @@ def get_answers_from_command_line(cml):
    answers["output:shared:out_dir"]    = os.path.abspath(cml.out_dir + '/')
    if  cml.merra2:
       init_merra2(answers)
-   else:   
+   else:
       answers["input:shared:bc_version"]   = cml.bcvin
       answers["input:surface:catch_model"] = cml.catch_model
       answers["input:shared:rst_dir"]      = os.path.abspath(cml.rst_dir + '/')
-      fvcore_info(answers) 
+      fvcore_info(answers)
       ogrid                            = cml.oceanin
       if ogrid == "CS":
          ogrid = answers["input:shared:agrid"]
@@ -144,7 +141,7 @@ def get_answers_from_command_line(cml):
 
    answers["output:air:agcm_import_rst"] = not cml.noagcm_import_rst
 
-   if cml.zoom: 
+   if cml.zoom:
       answers["input:surface:zoom"]    = cml.zoom
    else:
       # zoom_default fills 'input:shared:agrid'
@@ -163,7 +160,7 @@ def get_answers_from_command_line(cml):
    answers["slurm_pbs:account"]    = cml.account
    answers["slurm_pbs:qos"]        = cml.qos
    answers["slurm_pbs:partition"]  = cml.partition
-  
+
    return answers
 
 if __name__ == "__main__":
@@ -176,7 +173,7 @@ def get_answers_from_command_line(cml):
    with open("raw_command.yaml", "w") as f:
      yaml.dump(config, f)
 
-   config = get_config_from_answers(answers, config_tpl= True) 
+   config = get_config_from_answers(answers, config_tpl= True)
    with open("params_from_command.yaml", "w") as f:
      yaml.dump(config, f)
 
diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py
index f97712f9..65ac7c60 100755
--- a/pre/remap_restart/remap_questions.py
+++ b/pre/remap_restart/remap_questions.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 #
 # remap_restarts package:
-#   interactive questionary to create a yaml configuration file (remap_params.yaml) and 
+#   interactive questionary to create a yaml configuration file (remap_params.yaml) and
 #   a matching command line argument string (remap_restarts.CMD)
 #
 #
@@ -37,10 +37,7 @@ def echo_bcs(x,opt):
 
 def default_partition(x):
    if x['slurm_pbs:qos'] == 'debug':
-      if (BUILT_ON_SLES15):
-         x['slurm_pbs:partition'] = 'scutest'
-      else:
-         x['slurm_pbs:partition'] = 'compute'
+      x['slurm_pbs:partition'] = 'compute'
       return False
    return True
 
@@ -50,7 +47,7 @@ def validate_merra2_time(text):
       if hh in ['03','09','15','21']:
          return True
       else:
-         return False 
+         return False
    else:
      return False
 def SITE_MERRA2(x):
@@ -176,7 +173,7 @@ def ask_questions():
         {
             "type": "select",
             "name": "output:shared:stretch",
-            "message": message_stretch, 
+            "message": message_stretch,
             "choices": choices_stretch[1:3],
             "when": lambda x : x['output:shared:stretch'],
         },
@@ -185,7 +182,7 @@ def ask_questions():
             "type": "select",
             "name": "output:shared:agrid",
             "message": "Select resolution of SG001 grid for new restarts: \n",
-            "choices": choices_res_SG001, 
+            "choices": choices_res_SG001,
             "when": lambda x : x.get('output:shared:stretch') == 'SG001',
         },
 
@@ -193,7 +190,7 @@ def ask_questions():
             "type": "select",
             "name": "output:shared:agrid",
             "message": "Select resolution of SG002 grid for new restarts: \n",
-            "choices": choices_res_SG002, 
+            "choices": choices_res_SG002,
             "when": lambda x : x.get('output:shared:stretch') == 'SG002',
         },
 
@@ -344,8 +341,8 @@ def ask_questions():
         {
             "type": "confirm",
             "name": "output:air:agcm_import_rst",
-            "message": f'''Remap agcm_import_rst (a.k.a. IAU) file needed for REPLAY runs? 
-                        (NOTE: Preferred method is to regenerate IAU file, 
+            "message": f'''Remap agcm_import_rst (a.k.a. IAU) file needed for REPLAY runs?
+                        (NOTE: Preferred method is to regenerate IAU file,
                                but IF requested, remapping will be performed.)''',
             "default": False,
             "when": lambda x: echo_level(x),
@@ -430,13 +427,13 @@ def ask_questions():
    answers['output:shared:out_dir'] = os.path.abspath(answers['output:shared:out_dir'])
 
    if answers.get('input:air:nlevel') : del answers['input:air:nlevel']
-   if answers["output:surface:remap"] and not answers["input:shared:MERRA-2"]:  
+   if answers["output:surface:remap"] and not answers["input:shared:MERRA-2"]:
       answers["input:surface:catch_model"] = catch_model(answers)
    answers["output:surface:remap_water"] = answers["output:surface:remap"]
    answers["output:surface:remap_catch"] = answers["output:surface:remap"]
    del answers["output:surface:remap"]
    if answers["input:shared:MERRA-2"] : answers["input:air:hydrostatic"] = True
- 
+
    return answers
 
 if __name__ == "__main__":

From f2f4f3cd9cd03532323e405bedff5c9e3d1baeac Mon Sep 17 00:00:00 2001
From: Matthew Thompson <matthew.thompson@nasa.gov>
Date: Thu, 12 Sep 2024 11:47:51 -0400
Subject: [PATCH 10/24] Fix CI

---
 .circleci/config.yml | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index eeec7610..cd042ffd 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1,11 +1,11 @@
 version: 2.1
 
 # Anchors in case we need to override the defaults from the orb
-#baselibs_version: &baselibs_version v7.23.0
-#bcs_version: &bcs_version v11.5.0
+#baselibs_version: &baselibs_version v8.5.0
+#bcs_version: &bcs_version v12.0.0
 
 orbs:
-  ci: geos-esm/circleci-tools@2
+  ci: geos-esm/circleci-tools@5
 
 workflows:
   build-test:
@@ -21,7 +21,10 @@ workflows:
           #baselibs_version: *baselibs_version
           repo: GEOSgcm
           checkout_fixture: true
-          mepodevelop: true
+          # V12 code uses a special branch for now.
+          fixture_branch: feature/sdrabenh/gcm_v12
+          # We comment out this as it will "undo" the fixture_branch
+          #mepodevelop: true
           persist_workspace: true # Needs to be true to run fv3/gcm experiment, costs extra
 
       # Run AMIP GCM (1 hour, no ExtData)
@@ -45,7 +48,7 @@ workflows:
             - docker-hub-creds
           matrix:
             parameters:
-              compiler: [gfortran, ifort]
+              compiler: [ifort]
           requires:
             - build-GEOSgcm-on-<< matrix.compiler >>
           repo: GEOSgcm

From 45d03553dfb9f2d3408cd1bbbc453feca6e80fc6 Mon Sep 17 00:00:00 2001
From: biljanaorescanin <biljana.orescanin@nasa.gov>
Date: Fri, 3 Jan 2025 10:43:55 -0500
Subject: [PATCH 11/24] add missing commits

---
 pre/remap_restart/remap_command_line.py |  7 ++++++-
 pre/remap_restart/remap_questions.py    | 17 ++++++++++++++---
 pre/remap_restart/remap_utils.py        |  3 ++-
 3 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py
index 5475bf90..b9c1c6d9 100755
--- a/pre/remap_restart/remap_command_line.py
+++ b/pre/remap_restart/remap_command_line.py
@@ -81,7 +81,12 @@ def parse_args(program_description):
     p_command.add_argument('-qos',        default="",    help="slurm_pbs quality-of-service", choices=['', 'debug', 'allnccs', 'normal'])
     account = get_account()
     p_command.add_argument('-account',    default=account,    help="slurm_pbs account")
-    p_command.add_argument('-partition',  default='',       help="slurm_pbs partition")
+
+    if (BUILT_ON_SLES15):
+        p_command.add_argument('-partition',  default='scutest',help="slurm_pbs partition")
+    else:
+        p_command.add_argument('-partition',  default='',       help="slurm_pbs partition")
+
     p_command.add_argument('-rs',         default='3',        help="Flag indicating which restarts to regrid: 1 (upper air); 2 (surface); 3 (both)", choices=['1','2','3'])
 
     # Parse using parse_known_args so we can pass the rest to the remap scripts
diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py
index 020706f9..bc206b38 100755
--- a/pre/remap_restart/remap_questions.py
+++ b/pre/remap_restart/remap_questions.py
@@ -37,9 +37,12 @@ def echo_bcs(x,opt):
 
 def default_partition(x):
    if x['slurm_pbs:qos'] == 'debug':
-      x['slurm_pbs:partition'] = 'compute'
-      return False
-   return True
+       if (BUILT_ON_SLES15):
+           x['slurm_pbs:partition'] = 'scutest'
+           else:
+               x['slurm_pbs:partition'] = 'compute'
+               return False
+           return True
 
 def validate_merra2_time(text):
    if len(text) == 10 :
@@ -121,6 +124,13 @@ def ask_questions():
             "validate": lambda text: validate_geosit_time(text),
             "when": lambda x: x.get("input:shared:GEOS-IT", False) and not x.get("input:shared:MERRA-2", False),
         },
+        {
+            "type": "confirm",
+            "name": "input:air:hydrostatic",
+            "message": "Is the upper air input hydrostatic? (If you are not sure, don't change the default 'True')\n",
+            "default": True,
+            "when": lambda x: not x['input:shared:MERRA-2'],
+        },
         {
             "type": "path",
             "name": "output:shared:out_dir",
@@ -465,6 +475,7 @@ def ask_questions():
    answers["output:surface:remap_water"] = answers["output:surface:remap"]
    answers["output:surface:remap_catch"] = answers["output:surface:remap"]
    del answers["output:surface:remap"]
+   if answers["input:shared:MERRA-2"] : answers["input:air:hydrostatic"] = True
 
    return answers
 
diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py
index d385cc71..b3df9885 100755
--- a/pre/remap_restart/remap_utils.py
+++ b/pre/remap_restart/remap_utils.py
@@ -119,7 +119,8 @@
 
 message_ogrid_new   = "Select data ocean grid/resolution of output restarts:\n"
 
-message_qos        = "SLURM or PBS quality-of-service (qos)?  (If resolution is c1440 or higher, enter 'allnccs' for NCCS or 'normal' for NAS.)\n"
+message_qos        = "SLURM or PBS quality-of-service (qos)?      (Use default 'debug' to get resource faster; or
+     Enter 'allnccs' for NCCS or 'normal' for NAS if resolution is c1440 or higher; or leave it blank)\n"
 
 message_account    = "Select/enter SLURM or PBS account:\n"
 

From f704f6944bf2b50d347cdd14194dfa6ee95eb887 Mon Sep 17 00:00:00 2001
From: biljanaorescanin <biljana.orescanin@nasa.gov>
Date: Fri, 3 Jan 2025 11:27:19 -0500
Subject: [PATCH 12/24] fix the indentation

---
 pre/remap_restart/remap_questions.py | 14 +++++++-------
 pre/remap_restart/remap_utils.py     |  7 +++----
 2 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py
index bc206b38..b5e0a5c1 100755
--- a/pre/remap_restart/remap_questions.py
+++ b/pre/remap_restart/remap_questions.py
@@ -36,13 +36,13 @@ def echo_bcs(x,opt):
   return False
 
 def default_partition(x):
-   if x['slurm_pbs:qos'] == 'debug':
-       if (BUILT_ON_SLES15):
-           x['slurm_pbs:partition'] = 'scutest'
-           else:
-               x['slurm_pbs:partition'] = 'compute'
-               return False
-           return True
+    if x['slurm_pbs:qos'] == 'debug':
+        if BUILT_ON_SLES15:
+            x['slurm_pbs:partition'] = 'scutest'
+        else:
+            x['slurm_pbs:partition'] = 'compute'
+            return False
+        return True
 
 def validate_merra2_time(text):
    if len(text) == 10 :
diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py
index b3df9885..02377d9b 100755
--- a/pre/remap_restart/remap_utils.py
+++ b/pre/remap_restart/remap_utils.py
@@ -17,8 +17,8 @@
 
 #During cmake step, the string will be changed according to the system
 
-BUILT_ON_SLES15 = "@BUILT_ON_SLES15@"
-GEOS_SITE       = "@GEOS_SITE@"
+BUILT_ON_SLES15 = "TRUE"
+GEOS_SITE       = "NCCS"
 
 if BUILT_ON_SLES15 == "TRUE":
     BUILT_ON_SLES15 = True
@@ -119,8 +119,7 @@
 
 message_ogrid_new   = "Select data ocean grid/resolution of output restarts:\n"
 
-message_qos        = "SLURM or PBS quality-of-service (qos)?      (Use default 'debug' to get resource faster; or
-     Enter 'allnccs' for NCCS or 'normal' for NAS if resolution is c1440 or higher; or leave it blank)\n"
+message_qos        = "SLURM or PBS quality-of-service (qos)?      (Use default 'debug' to get resource faster; or Enter 'allnccs' for NCCS or 'normal' for NAS if resolution is c1440 or higher; or leave it blank)\n"
 
 message_account    = "Select/enter SLURM or PBS account:\n"
 

From 130e729d3b6629e0c45b6022cf1bc7a30ddec718 Mon Sep 17 00:00:00 2001
From: biljanaorescanin <biljana.orescanin@nasa.gov>
Date: Fri, 3 Jan 2025 12:09:41 -0500
Subject: [PATCH 13/24] typo

---
 pre/remap_restart/remap_utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py
index 02377d9b..dc782335 100755
--- a/pre/remap_restart/remap_utils.py
+++ b/pre/remap_restart/remap_utils.py
@@ -17,8 +17,8 @@
 
 #During cmake step, the string will be changed according to the system
 
-BUILT_ON_SLES15 = "TRUE"
-GEOS_SITE       = "NCCS"
+BUILT_ON_SLES15 = "@BUILT_ON_SLES15@"
+GEOS_SITE       = "@GEOS_SITE@"
 
 if BUILT_ON_SLES15 == "TRUE":
     BUILT_ON_SLES15 = True

From dd16e76688e7f9ed603bb40bb11c96556ede00ce Mon Sep 17 00:00:00 2001
From: biljanaorescanin <biljana.orescanin@nasa.gov>
Date: Fri, 3 Jan 2025 12:21:50 -0500
Subject: [PATCH 14/24] remove scutest

---
 pre/remap_restart/remap_command_line.py | 6 +-----
 pre/remap_restart/remap_questions.py    | 9 +++------
 2 files changed, 4 insertions(+), 11 deletions(-)

diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py
index b9c1c6d9..76677d16 100755
--- a/pre/remap_restart/remap_command_line.py
+++ b/pre/remap_restart/remap_command_line.py
@@ -81,11 +81,7 @@ def parse_args(program_description):
     p_command.add_argument('-qos',        default="",    help="slurm_pbs quality-of-service", choices=['', 'debug', 'allnccs', 'normal'])
     account = get_account()
     p_command.add_argument('-account',    default=account,    help="slurm_pbs account")
-
-    if (BUILT_ON_SLES15):
-        p_command.add_argument('-partition',  default='scutest',help="slurm_pbs partition")
-    else:
-        p_command.add_argument('-partition',  default='',       help="slurm_pbs partition")
+    p_command.add_argument('-partition',  default='',       help="slurm_pbs partition")
 
     p_command.add_argument('-rs',         default='3',        help="Flag indicating which restarts to regrid: 1 (upper air); 2 (surface); 3 (both)", choices=['1','2','3'])
 
diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py
index b5e0a5c1..23ee7ffc 100755
--- a/pre/remap_restart/remap_questions.py
+++ b/pre/remap_restart/remap_questions.py
@@ -37,12 +37,9 @@ def echo_bcs(x,opt):
 
 def default_partition(x):
     if x['slurm_pbs:qos'] == 'debug':
-        if BUILT_ON_SLES15:
-            x['slurm_pbs:partition'] = 'scutest'
-        else:
-            x['slurm_pbs:partition'] = 'compute'
-            return False
-        return True
+        x['slurm_pbs:partition'] = 'compute'
+        return False
+    return True
 
 def validate_merra2_time(text):
    if len(text) == 10 :

From 175d777a53580b0ce0c26625121c6d179b83a1fd Mon Sep 17 00:00:00 2001
From: Weiyuan Jiang <weiyuan.jiang@nasa.gov>
Date: Fri, 3 Jan 2025 16:17:02 -0500
Subject: [PATCH 15/24] add message for restart file names

---
 pre/remap_restart/remap_upper.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py
index 36c5a262..22074b39 100755
--- a/pre/remap_restart/remap_upper.py
+++ b/pre/remap_restart/remap_upper.py
@@ -423,7 +423,8 @@ def find_rst(self):
         if len(files) >0:
           restarts_in.append(files[0])
      if (len(restarts_in) == 0) :
-        print("\n try restart file names without time stamp\n")
+        print("\n Try restart file names without time stamp\n")
+        print("\n It expects restart file names as xx_internal_rst, e.g., fvcore_internal_rst \n")
         for f in self.air_restarts :
            fname = rst_dir+ '/'+f
            if os.path.exists(fname):

From 141ed37c56faf1a64c76d6105f5b349964556d54 Mon Sep 17 00:00:00 2001
From: biljanaorescanin <biljana.orescanin@nasa.gov>
Date: Mon, 6 Jan 2025 09:08:19 -0500
Subject: [PATCH 16/24] update user message

---
 pre/remap_restart/remap_upper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py
index 22074b39..211efed5 100755
--- a/pre/remap_restart/remap_upper.py
+++ b/pre/remap_restart/remap_upper.py
@@ -423,7 +423,7 @@ def find_rst(self):
         if len(files) >0:
           restarts_in.append(files[0])
      if (len(restarts_in) == 0) :
-        print("\n Try restart file names without time stamp\n")
+        print("\n Try restart file names without time stamp or suffix (e.g., .nc4)\n")
         print("\n It expects restart file names as xx_internal_rst, e.g., fvcore_internal_rst \n")
         for f in self.air_restarts :
            fname = rst_dir+ '/'+f

From ee4378fa1d33a61ee3b784f553e116671685721d Mon Sep 17 00:00:00 2001
From: William Putman <William.M.Putman@nasa.gov>
Date: Wed, 29 Jan 2025 12:04:23 -0500
Subject: [PATCH 17/24] fix for hydrostatic option in remap questions

---
 pre/remap_restart/remap_questions.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py
index 020706f9..40e0be40 100755
--- a/pre/remap_restart/remap_questions.py
+++ b/pre/remap_restart/remap_questions.py
@@ -121,6 +121,13 @@ def ask_questions():
             "validate": lambda text: validate_geosit_time(text),
             "when": lambda x: x.get("input:shared:GEOS-IT", False) and not x.get("input:shared:MERRA-2", False),
         },
+        {
+            "type": "confirm",
+            "name": "input:air:hydrostatic",
+            "message": "Is the upper air input hydrostatic? (If you are not sure, don't change the default 'True')\n",
+            "default": True,
+            "when": lambda x: not x['input:shared:MERRA-2'],
+        },
         {
             "type": "path",
             "name": "output:shared:out_dir",
@@ -465,6 +472,8 @@ def ask_questions():
    answers["output:surface:remap_water"] = answers["output:surface:remap"]
    answers["output:surface:remap_catch"] = answers["output:surface:remap"]
    del answers["output:surface:remap"]
+   if answers["input:shared:MERRA-2"] : answers["input:air:hydrostatic"] = True
+   if answers["input:shared:GEOS-IT"] : answers["input:air:hydrostatic"] = True
 
    return answers
 

From 3b561da26adf5ca882ef6a5ab1c7faa400960ec4 Mon Sep 17 00:00:00 2001
From: Matthew Thompson <matthew.thompson@nasa.gov>
Date: Tue, 11 Feb 2025 11:46:56 -0500
Subject: [PATCH 18/24] v12: Add support for v13 bcs in remap

---
 CHANGELOG.md                         |  5 +++--
 pre/remap_restart/remap_questions.py | 12 ++++++------
 pre/remap_restart/remap_utils.py     |  9 +++++----
 3 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2974a03b..8b851b41 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,9 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added choice 'blank' for qos in remap_restart python pacakage
 - Added hydrostatic question in remap_restart python pacakage
 - Added function to remap from stretched restarts to stretched restarts
-- Added remapping for GEOS-IT restarts 
-- Added new res C1120 
+- Added remapping for GEOS-IT restarts
+- Added new res C1120
 - NOTE: If running on SLES15 remap tests will not be zero diff for GOCART RST but are zero diff for all other
+- Add support for v13 BCs
 
 ### Changed
 
diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py
index 372044a8..6ba061c8 100755
--- a/pre/remap_restart/remap_questions.py
+++ b/pre/remap_restart/remap_questions.py
@@ -72,7 +72,7 @@ def SITE_GEOSIT(x):
   if GEOS_SITE == "NAS":
      x['input:shared:GEOS-IT']= False
      return False
-  return True 
+  return True
 
 def ask_questions():
 
@@ -161,7 +161,7 @@ def ask_questions():
            "when": lambda x: not x['input:shared:MERRA-2'] and not x['input:shared:GEOS-IT'] and not fvcore_info(x),
         },
 
-        {        
+        {
            "type": "select",
            "name": "input:shared:omodel",
            "message": "Select ocean model of input restarts:\n",
@@ -170,7 +170,7 @@ def ask_questions():
            "when": lambda x: not x['input:shared:MERRA-2'] and not x['input:shared:GEOS-IT'],
        },
 
-       {   
+       {
           "type": "select",
           "name": "input:shared:ogrid",
           "message": message_ogrid_in,
@@ -314,7 +314,7 @@ def ask_questions():
             "name": "output:shared:bc_version",
             "message": message_bc_other_new,
             "choices": choices_bc_other,
-            "when": lambda x:  x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] not in ['v06','v11','v12'],
+            "when": lambda x:  x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] not in ['v06','v11','v12','v13'],
         },
 
         {
@@ -322,7 +322,7 @@ def ask_questions():
             "name": "output:shared:bc_version",
             "message": "\nSelect BCs version for new restarts:\n",
             "choices": choices_bc_other,
-            "when": lambda x:  x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] in ['v06','v11','v12'],
+            "when": lambda x:  x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] in ['v06','v11','v12','v13'],
         },
 
         {
@@ -467,7 +467,7 @@ def ask_questions():
    answers['output:shared:out_dir'] = os.path.abspath(answers['output:shared:out_dir'])
 
    if answers.get('input:air:nlevel') : del answers['input:air:nlevel']
-   if answers["output:surface:remap"] and not answers["input:shared:MERRA-2"] and not answers["input:shared:GEOS-IT"]:  
+   if answers["output:surface:remap"] and not answers["input:shared:MERRA-2"] and not answers["input:shared:GEOS-IT"]:
       answers["input:surface:catch_model"] = catch_model(answers)
    answers["output:surface:remap_water"] = answers["output:surface:remap"]
    answers["output:surface:remap_catch"] = answers["output:surface:remap"]
diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py
index dc782335..16828171 100755
--- a/pre/remap_restart/remap_utils.py
+++ b/pre/remap_restart/remap_utils.py
@@ -36,9 +36,9 @@
 
 choices_bc_ops     = ['NL3', 'ICA', 'GM4', 'Other']
 
-choices_bc_other   = ['v06','v11','v12']
+choices_bc_other   = ['v06','v11','v12','v13']
 
-choices_bc_cmd     = ['NL3', 'ICA', 'GM4', 'v06', 'v11','v12']
+choices_bc_cmd     = ['NL3', 'ICA', 'GM4', 'v06', 'v11','v12', 'v13']
 
 choices_omodel     = ['data', 'MOM5', 'MOM6']
 
@@ -98,7 +98,8 @@
 
           v06:     NL3 + JPL veg height + PEATMAP + MODIS snow alb\n
           v11:     NL3 + JPL veg height + PEATMAP + MODIS snow alb v2\n
-          v12:     NL3 + JPL veg height + PEATMAP + MODIS snow alb v2 + Argentina peatland fix \n\n'''\
+          v12:     NL3 + JPL veg height + PEATMAP + MODIS snow alb v2 + Argentina peatland fix \n
+          v13:     NL3 + JPL veg height + PEATMAP + MODIS snow alb v2 + Argentina peatland fix + mean land elevation fix \n\n'''\
 
 message_bc_other_in  = ("Select BCs version of input restarts:\n" + message_bc_other)
 message_bc_other_new = ("Select BCs version for new restarts:\n"  + message_bc_other)
@@ -455,7 +456,7 @@ def get_command_line_from_answers(answers):
 
    nobkg  = '' if answers["output:analysis:bkg"] else " -nobkg "
    nolcv  = '' if answers["output:analysis:lcv"] else " -nolcv "
-   nonhydrostatic = '' if answers["input:air:hydrostatic"] else " -nonhydrostatic " 
+   nonhydrostatic = '' if answers["input:air:hydrostatic"] else " -nonhydrostatic "
    label  = ' -lbl ' if answers["output:shared:label"] else ""
 
    in_bc_base  = ' -in_bc_base '  + answers.get("input:shared:bc_base")

From 553a82026adfba82c10ec733f43a7359d2aae55b Mon Sep 17 00:00:00 2001
From: Matthew Thompson <matthew.thompson@nasa.gov>
Date: Tue, 25 Mar 2025 11:51:02 -0400
Subject: [PATCH 19/24] v12: Remove NCCS OS detection

---
 pre/remap_restart/remap_catchANDcn.py | 50 +++++++++++++--------------
 pre/remap_restart/remap_upper.py      | 26 +++++++-------
 pre/remap_restart/remap_utils.py      |  6 ----
 3 files changed, 36 insertions(+), 46 deletions(-)

diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py
index cc642757..0ec9247f 100755
--- a/pre/remap_restart/remap_catchANDcn.py
+++ b/pre/remap_restart/remap_catchANDcn.py
@@ -44,8 +44,8 @@ def remap(self):
         in_rstfiles = glob.glob(rst_dir+'/*'+model+'_*')
      if len(in_rstfiles) == 0:
         return
-     in_rstfile = in_rstfiles[0]  
-             
+     in_rstfile = in_rstfiles[0]
+
 
      cwdir          = os.getcwd()
      bindir         = os.path.dirname(os.path.realpath(__file__))
@@ -60,8 +60,8 @@ def remap(self):
      surflay        = config['output']['surface']['surflay']
      in_tilefile    = config['input']['surface']['catch_tilefile']
 
-     if "gmao_SIteam/ModelData" in out_bc_base: 
-        assert  GEOS_SITE == "NAS", "wrong site to run the package" 
+     if "gmao_SIteam/ModelData" in out_bc_base:
+        assert  GEOS_SITE == "NAS", "wrong site to run the package"
 
      if not in_tilefile :
         agrid        = config['input']['shared']['agrid']
@@ -86,7 +86,7 @@ def remap(self):
 
      out_bc_landdir = get_landdir(out_bc_base, out_bc_version, agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch, grid=EASE_grid)
 
-     label = get_label(config) 
+     label = get_label(config)
 
      suffix     = time+'z.nc4' + label
 
@@ -100,7 +100,7 @@ def remap(self):
 
      print("\nRemapping " + model + ".....\n")
  # determine NPE based on *approximate* number of input and output tile
-      
+
      in_Ntile  = 0
      mime = mimetypes.guess_type(in_rstfile)
      if mime[0] and 'stream' in mime[0]: # binary
@@ -108,14 +108,14 @@ def remap(self):
      else : # nc4 file
        ds = nc.Dataset(in_rstfile)
        in_Ntile = ds.dimensions['tile'].size
-       
+
      out_Ntile = 0
      with open( out_bc_landdir+'/clsm/catchment.def') as f:
        out_Ntile = int(next(f))
      max_Ntile = max(in_Ntile, out_Ntile)
      NPE = 0
      if   (max_Ntile <=  112573) : # no more than EASEv2_M36
-        NPE = 40      
+        NPE = 40
      elif (max_Ntile <= 1684725) : # no more than EASEv2_M09
         NPE = 80
      elif (max_Ntile <= 2496756) : # no more than C720
@@ -144,9 +144,7 @@ def remap(self):
        if (partition != ''):
          PARTITION = "#SBATCH --partition=" + partition
 
-       CONSTRAINT = '"[cas|sky]"'
-       if BUILT_ON_SLES15:
-         CONSTRAINT = 'mil'
+       CONSTRAINT = '"[cas|mil]"'
        if (qos != ''):
          QOS = "#SBATCH  --qos="+qos
 
@@ -163,7 +161,7 @@ def remap(self):
      InData_dir = out_dir+'/InData/'
      print ("mkdir -p " + InData_dir)
      os.makedirs(InData_dir, exist_ok = True)
-    
+
      f = os.path.basename(in_rstfile)
      dest = InData_dir+'/'+f
      # file got copy because the computing node cannot access archive
@@ -183,8 +181,8 @@ def remap(self):
 
 set params = ( -model {model}  -time {time} -in_tilefile {in_tilefile} )
 set params = ( $params -out_bcs {out_bcs} -out_tilefile {out_tilefile} -out_dir {out_dir} )
-set params = ( $params -surflay {surflay} -in_wemin {in_wemin} -out_wemin {out_wemin} ) 
-set params = ( $params -in_rst {in_rstfile} -out_rst {out_rstfile} ) 
+set params = ( $params -surflay {surflay} -in_wemin {in_wemin} -out_wemin {out_wemin} )
+set params = ( $params -in_rst {in_rstfile} -out_rst {out_rstfile} )
 $esma_mpirun_X $mk_catchANDcnRestarts_X $params
 
 """
@@ -263,7 +261,7 @@ def ask_catch_questions():
    catch_input_shared_rst_dir = ''
    def has_rs_rc_out(path):
      if os.path.exists(path):
-        dirs = os.listdir(path)   
+        dirs = os.listdir(path)
         if 'rs' in dirs and 'rc_out' in dirs:
            nonlocal catch_input_shared_rst_dir
            catch_input_shared_rst_dir = path
@@ -280,7 +278,7 @@ def has_catch_rst(text):
      rst_dir   = catch_input_shared_rst_dir + '/rs/ens0000/Y'+yyyy +'/M'+mm+'/'
      rst_files = glob.glob(rst_dir+'*catch*_internal_rst.'+yyyy+mm+dd+'_'+hh+'00')
      if len(rst_files) !=1 :
-       return False 
+       return False
      return True
 
    questions =[
@@ -297,11 +295,11 @@ def has_catch_rst(text):
             "message": (message_datetime + " and rst file must exist.)\n"),
             "validate": lambda text: has_catch_rst(text)
         },
-        
+
         {
             "type": "path",
             "name": "output:shared:out_dir",
-            "message": message_out_dir, 
+            "message": message_out_dir,
         },
 
         {
@@ -309,7 +307,7 @@ def has_catch_rst(text):
             "name": "output:shared:expid",
             "message": message_expid,
             "default": "",
-        },  
+        },
 
         {
             "type": "select",
@@ -326,7 +324,7 @@ def has_catch_rst(text):
             "choices": choices_bc_ops,
             "default": "NL3",
         },
-        {   
+        {
             "type": "select",
             "name": "output:shared:bc_version",
             "message": message_bc_other_new,
@@ -334,7 +332,7 @@ def has_catch_rst(text):
             "when": lambda x:  x["output:shared:bc_version"] == 'Other',
         },
 
-        {   
+        {
             "type": "select",
             "name": "output:surface:EASE_grid",
             "message": "Select grid for new restart:\n",
@@ -387,29 +385,29 @@ def has_catch_rst(text):
             "name": "slurm_pbs:partition",
             "message": message_partition,
             "default": '',
-        },  
+        },
 
    ]
 
-   answers = questionary.prompt(questions)   
+   answers = questionary.prompt(questions)
    yyyy = answers['input:shared:yyyymmddhh'][0:4]
    mm   = answers['input:shared:yyyymmddhh'][4:6]
    dd   = answers['input:shared:yyyymmddhh'][6:8]
    hh   = answers['input:shared:yyyymmddhh'][8:10]
-  
+
    rst_dir      = answers['input:shared:rst_dir']+'/rs/ens0000/Y'+yyyy +'/M'+mm+'/'
    rst_file     = glob.glob(rst_dir+'*catch*_internal_rst.'+yyyy+mm+dd+'_'+hh+'00')[0]
    idx1         = rst_file.find('catch')
    idx2         = rst_file.find('_internal_rst')
    catch_model  = rst_file[idx1:idx2]
-   
+
    tile_dir     = answers['input:shared:rst_dir']+'/rc_out/'
    in_tilefiles = glob.glob(tile_dir+'*.til')
    for file in in_tilefiles:
       answers['input:surface:catch_tilefile'] = file
       if 'MAPL_' in file:
          answers['input:surface:catch_tilefile'] = file
-       
+
    answers['input:shared:rst_dir'] = rst_dir
    answers['input:surface:catch_model'] =  catch_model
    answers['input:surface:wemin']  = "13"
diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py
index 211efed5..b7266c4a 100755
--- a/pre/remap_restart/remap_upper.py
+++ b/pre/remap_restart/remap_upper.py
@@ -78,9 +78,9 @@ def remap(self):
      label = get_label(config)
      suffix = yyyymmddhh_[0:8]+'_'+yyyymmddhh_[8:10] +'z' + types + label
 
-     in_bc_base     = config['input']['shared']['bc_base'] 
+     in_bc_base     = config['input']['shared']['bc_base']
      if "gmao_SIteam/ModelData" in in_bc_base:
-        assert  GEOS_SITE == "NAS", "wrong site to run the package" 
+        assert  GEOS_SITE == "NAS", "wrong site to run the package"
 
      in_bc_version  = config['input']['shared']['bc_version']
      agrid       = config['input']['shared']['agrid']
@@ -90,7 +90,7 @@ def remap(self):
      topo_bcsdir = get_topodir(in_bc_base, in_bc_version,  agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch)
      topoin = glob.glob(topo_bcsdir+'/topo_DYN_ave*.data')[0]
 
-     out_bc_base    = config['output']['shared']['bc_base'] 
+     out_bc_base    = config['output']['shared']['bc_base']
      out_bc_version = config['output']['shared']['bc_version']
      agrid       = config['output']['shared']['agrid']
      ogrid       = config['output']['shared']['ogrid']
@@ -125,7 +125,7 @@ def remap(self):
        cmd = '/bin/ln -s  ' + rst + ' ' + f
        print('\n'+cmd)
        subprocess.call(shlex.split(cmd))
- 
+
      # link topo file
 
      cmd = '/bin/ln -s ' + topoin + ' .'
@@ -163,7 +163,7 @@ def remap(self):
      QOS       =''
      qos       = config['slurm_pbs']['qos']
      TIME = "1:00:00"
-     if NPE > 532: 
+     if NPE > 532:
         assert config['slurm_pbs']['qos'] != 'debug', "qos should be 'allnccs' for NCCS or 'normal' for NAS"
         TIME = "3:00:00"
      NNODE = ''
@@ -182,9 +182,7 @@ def remap(self):
        if (qos != ''):
          QOS = "#SBATCH  --qos="+qos
 
-       CONSTRAINT = '"[cas|sky]"'
-       if BUILT_ON_SLES15:
-         CONSTRAINT = 'mil'
+       CONSTRAINT = '"[cas|mil]"'
 
      # We need to create an input.nml file which is different if we are running stretched grid
      # If we are running stretched grid, we need to pass in the target lon+lat and stretch factor
@@ -205,7 +203,7 @@ def remap(self):
         else:
           exit("This stretched grid option is not supported " + str(stretch))
 
-        # note "reversed" order of args (relative to order in definition of STRETCH_GRID)  
+        # note "reversed" order of args (relative to order in definition of STRETCH_GRID)
 
         stretch_str = "-stretched_grid_in " + str(target_lon) + " " + str(target_lat) + " " + str(stretch_fac)
 
@@ -221,9 +219,9 @@ def remap(self):
           target_lon  = STRETCH_GRID['SG002'][2]
         else:
           exit("This stretched grid option is not supported " + str(stretch))
-     
-        # note "reversed" order of args (relative to order in definition of STRETCH_GRID)  
-     
+
+        # note "reversed" order of args (relative to order in definition of STRETCH_GRID)
+
         stretch_str = stretch_str+" -stretched_grid_out " + str(target_lon) + " " + str(target_lat) + " " + str(stretch_fac)
 
      # Now, let's create the input.nml file
@@ -248,7 +246,7 @@ def remap(self):
 /bin/touch input.nml
 
 set mpi_type = "openmpi"
-if ($?I_MPI_ROOT ) then 
+if ($?I_MPI_ROOT ) then
   set mpi_type = "intel"
 endif
 
@@ -341,7 +339,7 @@ def remap(self):
 endif
 
 {Bin}/esma_mpirun -np {NPE} $interp_restartsX -im {imout} -lm {nlevel} \\
-   $hydrostaticflag {stretch_str} $dmflag -nwriter {nwrit} $ioflag 
+   $hydrostaticflag {stretch_str} $dmflag -nwriter {nwrit} $ioflag
 
 """
      account = config['slurm_pbs']['account']
diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py
index 16828171..67c570a2 100755
--- a/pre/remap_restart/remap_utils.py
+++ b/pre/remap_restart/remap_utils.py
@@ -17,14 +17,8 @@
 
 #During cmake step, the string will be changed according to the system
 
-BUILT_ON_SLES15 = "@BUILT_ON_SLES15@"
 GEOS_SITE       = "@GEOS_SITE@"
 
-if BUILT_ON_SLES15 == "TRUE":
-    BUILT_ON_SLES15 = True
-else:
-    BUILT_ON_SLES15 = False
-
 # top-level directory for BCs (machine-dependent)
 
 choices_bc_base  =[ "NCCS/Discover : /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles",

From 9ebc1805ef1fd21c44ce625d4bd88f7c61f3ea52 Mon Sep 17 00:00:00 2001
From: Matthew Thompson <matthew.thompson@nasa.gov>
Date: Tue, 25 Mar 2025 11:51:54 -0400
Subject: [PATCH 20/24] Update changelog

---
 CHANGELOG.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8b851b41..f4f49c2e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -25,6 +25,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Removed
 
+- Removed detection of OS at NCCS as it is all SLES15 now
+
 ### Deprecated
 
 ## [2.1.6] - 2025-01-23

From 40e04763e57ad15183e682a4f1aec9f3c637fa35 Mon Sep 17 00:00:00 2001
From: Shayon Shakoorzadeh <shayon.shakoorzadeh@nasa.gov>
Date: Mon, 14 Apr 2025 15:30:46 -0400
Subject: [PATCH 21/24] resolved issue #131 in gcm_v12

---
 pre/remap_restart/remap_questions.py | 10 +++++-----
 pre/remap_restart/remap_utils.py     | 20 +++++++++-----------
 2 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py
index 6ba061c8..cc04c2b0 100755
--- a/pre/remap_restart/remap_questions.py
+++ b/pre/remap_restart/remap_questions.py
@@ -271,7 +271,7 @@ def ask_questions():
             "type": "text",
             "name": "output:air:nlevel",
             "message": "Enter number of atmospheric levels for new restarts: (71 72 91 127 132 137 144 181)\n",
-            "default": "72",
+            "default": "181",
         },
 
         # to show the message, we ask output first
@@ -296,7 +296,7 @@ def ask_questions():
             "name": "output:shared:bc_version",
             "message": message_bc_ops_new,
             "choices": choices_bc_ops,
-            "default": "NL3",
+            "default": "v13",
             "when": lambda x: x["input:shared:MERRA-2"] or x["input:shared:GEOS-IT"],
         },
 
@@ -305,7 +305,7 @@ def ask_questions():
             "name": "output:shared:bc_version",
             "message": "Select BCs version for new restarts:\n",
             "choices": choices_bc_ops,
-            "default": "NL3",
+            "default": "v13",
             "when": lambda x: not x["input:shared:MERRA-2"] and not x["input:shared:GEOS-IT"],
         },
 
@@ -314,7 +314,7 @@ def ask_questions():
             "name": "output:shared:bc_version",
             "message": message_bc_other_new,
             "choices": choices_bc_other,
-            "when": lambda x:  x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] not in ['v06','v11','v12','v13'],
+            "when": lambda x:  x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] not in ['v06','v11','v12','GM4'],
         },
 
         {
@@ -322,7 +322,7 @@ def ask_questions():
             "name": "output:shared:bc_version",
             "message": "\nSelect BCs version for new restarts:\n",
             "choices": choices_bc_other,
-            "when": lambda x:  x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] in ['v06','v11','v12','v13'],
+            "when": lambda x:  x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] in ['v06','v11','v12','GM4'],
         },
 
         {
diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py
index 67c570a2..99ab3866 100755
--- a/pre/remap_restart/remap_utils.py
+++ b/pre/remap_restart/remap_utils.py
@@ -28,9 +28,9 @@
 # define "choices", "message" strings, and "validate" lists that are used multiple times
 #   (and related definitions, even if they are used just once).
 
-choices_bc_ops     = ['NL3', 'ICA', 'GM4', 'Other']
+choices_bc_ops     = ['v13', 'NL3', 'ICA', 'Other']
 
-choices_bc_other   = ['v06','v11','v12','v13']
+choices_bc_other   = ['v06','v11','v12','GM4']
 
 choices_bc_cmd     = ['NL3', 'ICA', 'GM4', 'v06', 'v11','v12', 'v13']
 
@@ -78,22 +78,20 @@
 message_bc_ops     = f'''\n
  BCs version      | ADAS tags            | GCM tags typically used with BCs version
  -----------------|----------------------|-----------------------------------------
- GM4: Ganymed-4_0 | 5_12_2 ... 5_16_5    | Ganymed-4_0      ... Heracles-5_4_p3
+ v13: v13         | future               | 12.0             ... present
+ NL3: Icarus-NLv3 | 5_25_1 ... present   | Icarus_NL, 10.19 ... 11.7
  ICA: Icarus      | 5_17_0 ... 5_24_0_p1 | Icarus, Jason    ... 10.18
- NL3: Icarus-NLv3 | 5_25_1 ... present   | Icarus_NL, 10.19 ... present
  ----------------------------------------------------------------------------------
- Other: Additional choices used in model or DAS development.
-           \n\n '''
+ Other: Additional choices used in model or DAS development.\n\n'''
 
 message_bc_ops_in  = ("Select boundary conditions (BCs) version of input restarts:\n" + message_bc_ops)
 message_bc_ops_new = ("Select boundary conditions (BCs) version for new restarts:\n"  + message_bc_ops)
 
 message_bc_other   = f'''\n
-
-          v06:     NL3 + JPL veg height + PEATMAP + MODIS snow alb\n
-          v11:     NL3 + JPL veg height + PEATMAP + MODIS snow alb v2\n
-          v12:     NL3 + JPL veg height + PEATMAP + MODIS snow alb v2 + Argentina peatland fix \n
-          v13:     NL3 + JPL veg height + PEATMAP + MODIS snow alb v2 + Argentina peatland fix + mean land elevation fix \n\n'''\
+          v06:     NL3 + JPL veg height + PEATMAP + MODIS snow alb
+          v11:     NL3 + JPL veg height + PEATMAP + MODIS snow alb v2
+          v12:     NL3 + JPL veg height + PEATMAP + MODIS snow alb v2 + Argentina peatland fix
+          GM4:     Ganymed-4_0\n'''\
 
 message_bc_other_in  = ("Select BCs version of input restarts:\n" + message_bc_other)
 message_bc_other_new = ("Select BCs version for new restarts:\n"  + message_bc_other)

From 248f714f0e13298f89e7df32612dcd225b6459f9 Mon Sep 17 00:00:00 2001
From: Matthew Thompson <matthew.thompson@nasa.gov>
Date: Tue, 15 Apr 2025 12:25:35 -0400
Subject: [PATCH 22/24] Fix bug with MERRA2 remapping

---
 pre/remap_restart/remap_questions.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py
index cc04c2b0..618e956c 100755
--- a/pre/remap_restart/remap_questions.py
+++ b/pre/remap_restart/remap_questions.py
@@ -6,12 +6,8 @@
 #
 #
 import os
-import subprocess
-import shlex
 import ruamel.yaml
-import shutil
 import questionary
-import glob
 from remap_utils import *
 
 def echo_level(x):
@@ -466,14 +462,21 @@ def ask_questions():
    answers['input:shared:rst_dir']  = os.path.abspath(answers['input:shared:rst_dir'])
    answers['output:shared:out_dir'] = os.path.abspath(answers['output:shared:out_dir'])
 
-   if answers.get('input:air:nlevel') : del answers['input:air:nlevel']
+   if answers.get('input:air:nlevel'):
+       del answers['input:air:nlevel']
    if answers["output:surface:remap"] and not answers["input:shared:MERRA-2"] and not answers["input:shared:GEOS-IT"]:
       answers["input:surface:catch_model"] = catch_model(answers)
    answers["output:surface:remap_water"] = answers["output:surface:remap"]
    answers["output:surface:remap_catch"] = answers["output:surface:remap"]
    del answers["output:surface:remap"]
-   if answers["input:shared:MERRA-2"] : answers["input:air:hydrostatic"] = True
-   if answers["input:shared:GEOS-IT"] : answers["input:air:hydrostatic"] = True
+   if answers["input:shared:MERRA-2"]:
+       answers["input:air:hydrostatic"] = True
+       # Due to the order of questions above, if a user asks
+       # for MERRA2, they will not be asked for GEOS-IT so
+       # we set to false so the next if-block doesn't run
+       answers["input:shared:GEOS-IT"] = False
+   if answers["input:shared:GEOS-IT"]:
+       answers["input:air:hydrostatic"] = True
 
    return answers
 

From 1d687047aefc01f66f399a65499ff77ce4525ff2 Mon Sep 17 00:00:00 2001
From: Matthew Thompson <matthew.thompson@nasa.gov>
Date: Fri, 13 Jun 2025 13:05:47 -0400
Subject: [PATCH 23/24] v12: Fix saltsplit bug

---
 .../remap_lake_landice_saltwater.py           | 21 +++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/pre/remap_restart/remap_lake_landice_saltwater.py b/pre/remap_restart/remap_lake_landice_saltwater.py
index b8b26988..a2d03449 100755
--- a/pre/remap_restart/remap_lake_landice_saltwater.py
+++ b/pre/remap_restart/remap_lake_landice_saltwater.py
@@ -139,10 +139,12 @@ def remap(self):
        cmd = exe + out_til + ' ' + in_til + ' InData/'+ saltwater_internal + ' 0 ' + str(zoom)
        self.run_and_log(cmd, log_name)
   
-       # split Saltwater
-       if  config['output']['surface']['split_saltwater']:
-         print("\nSplitting Saltwater...\n")
-         cmd = bindir+'/SaltIntSplitter.x ' + out_til + ' ' + 'OutData/' + saltwater
+       # split Saltwater Internal
+       # NOTE: split_saltwater==True means that the input restarts are already split.
+       #       So we do not split them again.
+       if not config['output']['surface']['split_saltwater']:
+         print("\nSplitting Saltwater Internal...\n")
+         cmd = bindir+'/SaltIntSplitter.x ' + out_til + ' ' + 'OutData/' + saltwater_internal
 #         subprocess.call(shlex.split(cmd))
          openwater = ''
          seaice  = ''
@@ -152,6 +154,17 @@ def remap(self):
        cmd = exe + out_til + ' ' + in_til + ' InData/'+ saltwater_import + ' 0 ' + str(zoom)
        self.run_and_log(cmd, log_name)
 
+       # split Saltwater Import
+       # NOTE: split_saltwater==True means that the input restarts are already split.
+       #       So we do not split them again.
+       if not config['output']['surface']['split_saltwater']:
+         print("\nSplitting Saltwater Import...\n")
+         cmd = bindir+'/SaltIntSplitter.x ' + out_til + ' ' + 'OutData/' + saltwater_import
+#         subprocess.call(shlex.split(cmd))
+         openwater = ''
+         seaice  = ''
+         self.run_and_log(cmd, log_name)
+
      if (openwater):
        cmd = exe + out_til + ' ' + in_til + ' InData/' + openwater + ' 0 ' + str(zoom)
        self.run_and_log(cmd, log_name)

From bcd66fc9441e7bbc5d08bcdad93a88edff3815db Mon Sep 17 00:00:00 2001
From: Shayon Shakoorzadeh <shayon.shakoorzadeh@nasa.gov>
Date: Tue, 29 Jul 2025 13:09:25 -0400
Subject: [PATCH 24/24] confocn.py and plotocn.j jinja compatability

---
 coupled_diagnostics/g5lib/confocn.py | 12 ++++++------
 coupled_diagnostics/g5lib/plotocn.j  | 10 +++++-----
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/coupled_diagnostics/g5lib/confocn.py b/coupled_diagnostics/g5lib/confocn.py
index 89f6ada6..4484a422 100644
--- a/coupled_diagnostics/g5lib/confocn.py
+++ b/coupled_diagnostics/g5lib/confocn.py
@@ -1,17 +1,17 @@
 '''
-@EXPDSC
+{{ EXPDSC }}
 
 ### Put detailed experiment description here ###
 '''
 import os
 from datetime import date
 
-expid='@EXPID'
-cmpexp='@EXPID'
-data_path='@EXPDIR'
+expid='{{ EXPID }}'
+cmpexp='{{ EXPID }}'
+data_path='{{ EXPDIR }}'
 plot_path=data_path+'/plots_ocn'
-basin_mask='@COUPLEDIR/basin_mask/@OGCM_IMx@OGCM_JM/basin_mask.nc'
-grid_spec='@COUPLEDIR/a@AGCM_IMx@AGCM_JM_o@OGCM_IMx@OGCM_JM/INPUT/grid_spec.nc'
+basin_mask='{{ COUPLEDIR }}/basin_mask/{{ OGCM_IM }}x{{ OGCM_JM }}/basin_mask.nc'
+grid_spec='{{ COUPLEDIR }}/a{{ AGCM_IM }}x{{ AGCM_JM }}_o{{ OGCM_IM }}x{{ OGCM_JM }}/INPUT/grid_spec.nc'
 
 start_year=1980
 end_year=1981
diff --git a/coupled_diagnostics/g5lib/plotocn.j b/coupled_diagnostics/g5lib/plotocn.j
index 1c4fd213..ce9b3db8 100755
--- a/coupled_diagnostics/g5lib/plotocn.j
+++ b/coupled_diagnostics/g5lib/plotocn.j
@@ -1,5 +1,5 @@
 #!/bin/csh 
-#SBATCH --job-name=@PLOT_T
+#SBATCH --job-name={{ PLOT_T }}
 #SBATCH --ntasks=20
 #SBATCH --time=12:00:00 
 #SBATCH -o plotocn.out
@@ -7,7 +7,7 @@
 source /usr/share/modules/init/csh
 module purge
 module load python/GEOSpyD/Ana2019.10_py2.7
-set SCRDIR=@GEOSDIR/coupled_diagnostics
+set SCRDIR={{ GEOSDIR }}/coupled_diagnostics
 set G5LIBDIR=$SCRDIR
 set ANADIR=$SCRDIR/analysis
 set VERIFICATION=$SCRDIR/verification
@@ -18,9 +18,9 @@ else if ( ($HOSTNAME =~ "pfe"*) || ($HOSTNAME =~ r[0-9]*i[0-9]*n[0-9]*) || ($HOS
     setenv OCEANVAL /nobackup/gmao_SIteam/ModelData/oceanval/verification
 endif 
 
-set EXPID=@EXPID
-set HOMDIR=@HOMDIR
-set EXPDIR=@EXPDIR
+set EXPID={{ EXPID }}
+set HOMDIR={{ HOMDIR }}
+set EXPDIR={{ EXPDIR }}
 set WORKDIR=$EXPDIR/plot
 
 setenv PYTHONPATH ${G5LIBDIR}:${VERIFICATION}:${HOMDIR}/..