From 1d831ae5ec24e6e2c63ff51296460333c7dac25e Mon Sep 17 00:00:00 2001 From: biljanaorescanin Date: Thu, 18 Sep 2025 14:19:57 -0400 Subject: [PATCH 1/9] add v14 bcs and update topo path --- pre/remap_restart/remap_questions.py | 4 +-- pre/remap_restart/remap_utils.py | 37 +++++++++++++++++++--------- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 481c3f0..44f42f3 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -308,7 +308,7 @@ def ask_questions(): "name": "output:shared:bc_version", "message": message_bc_other_new, "choices": choices_bc_other, - "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] not in ['v06','v11','v12'], + "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] not in ['v06','v11','v12','v14'], }, { @@ -316,7 +316,7 @@ def ask_questions(): "name": "output:shared:bc_version", "message": "\nSelect BCs version for new restarts:\n", "choices": choices_bc_other, - "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] in ['v06','v11','v12'], + "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] in ['v06','v11','v12','v14'], }, { diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 783302a..cbe58ee 100755 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -13,6 +13,7 @@ import shlex import netCDF4 as nc import linecache +import re # shared global variables @@ -31,9 +32,9 @@ choices_bc_ops = ['NL3', 'ICA', 'GM4', 'Other'] -choices_bc_other = ['v06','v11','v12'] +choices_bc_other = ['v06','v11','v12','v14'] -choices_bc_cmd = ['NL3', 'ICA', 'GM4', 'v06', 'v11','v12'] +choices_bc_cmd = ['NL3', 'ICA', 'GM4', 'v06', 'v11','v12', 'v14'] choices_omodel = ['data', 'MOM5', 'MOM6'] @@ -93,7 +94,8 @@ v06: NL3 + JPL veg height + PEATMAP + MODIS snow alb\n v11: NL3 + JPL veg height + PEATMAP + MODIS snow alb v2\n - v12: NL3 + JPL veg height + PEATMAP + MODIS snow alb v2 + Argentina peatland fix \n\n'''\ + v12: NL3 + JPL veg height + PEATMAP + MODIS snow alb v2 + Argentina peatland fix \n + v14: v12 + Coupled MOM6/v2 ocean bathymetry (OM4) and v2 topography for atmosphere \n\n'''\ message_bc_other_in = ("Select BCs version of input restarts:\n" + message_bc_other) message_bc_other_new = ("Select BCs version for new restarts:\n" + message_bc_other) @@ -618,16 +620,29 @@ def get_default_bc_base(): return choices_bc_base[0] return choices_bc_base[1] + def get_topodir(bc_base, bc_version, agrid=None, ogrid=None, omodel=None, stretch=None): - gridStr = get_resolutions(agrid=agrid, ogrid=ogrid, omodel=omodel,stretch=stretch) - agrid_name = gridStr.split('_')[0] - bc_topo = '' - if 'GM4' == bc_version: - bc_topo = bc_base + '/' + bc_version + '/TOPO/TOPO_' + agrid_name - else: - bc_topo = bc_base + '/' + bc_version + '/TOPO/TOPO_' + agrid_name + '/smoothed' + gridStr = get_resolutions(agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch) + agrid_name = gridStr.split('_')[0] + + v = str(bc_version).strip().upper() + + # GM4 stays on legacy tiles tree + if v.startswith('GM4'): + return os.path.join(str(bc_base), 'GM4', 'TOPO', f'TOPO_{agrid_name}') + + # v2 from v14+, else v1 + m = re.search(r'(\d+)', v) + ver_num = int(m.group(1)) if m else None + stream = 'v2' if (ver_num is not None and ver_num >= 14) else 'v1' + + # Derive new topo base from bc_base; fallback to default if pattern not found + bc_base_str = str(bc_base or '') + topo_base = re.sub(r'/fvInput/ExtData/esm/tiles/?$', '/make_bcs_inputs/atmosphere', bc_base_str) + if topo_base == bc_base_str or not topo_base: + topo_base = '/discover/nobackup/projects/gmao/bcs_shared/make_bcs_inputs/atmosphere' - return bc_topo + return os.path.join(topo_base, 'TOPO', stream, agrid_name, 'smoothed') def get_landdir(bc_base, bc_version, agrid=None, ogrid=None, omodel=None, stretch=None, grid=None): gridStr = get_resolutions(agrid=agrid, ogrid=ogrid, omodel=omodel,stretch=stretch, grid=grid) From e79f6614623a7e8039a7c8d92fb899799b5e70a2 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Fri, 24 Oct 2025 08:47:17 -0400 Subject: [PATCH 2/9] Fix bad merge that disappeared GM4 --- pre/remap_restart/remap_questions.py | 4 ++-- pre/remap_restart/remap_utils.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 47a525f..23b3d5d 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -310,7 +310,7 @@ def ask_questions(): "name": "output:shared:bc_version", "message": message_bc_other_new, "choices": choices_bc_other, - "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] not in ['v06','v11','v12','v14'], + "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] not in ['v06','v11','v12','v14','GM4'], }, { @@ -318,7 +318,7 @@ def ask_questions(): "name": "output:shared:bc_version", "message": "\nSelect BCs version for new restarts:\n", "choices": choices_bc_other, - "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] in ['v06','v11','v12','v14'], + "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] in ['v06','v11','v12','v14','GM4'], }, { diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 61b312f..6dc82a8 100755 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -32,7 +32,7 @@ choices_bc_ops = ['v13', 'NL3', 'ICA', 'Other'] -choices_bc_other = ['v06','v11','v12','v14'] +choices_bc_other = ['v06','v11','v12','v14','GM4'] choices_bc_cmd = ['NL3', 'ICA', 'GM4', 'v06', 'v11','v12', 'v14'] @@ -94,7 +94,9 @@ v06: NL3 + JPL veg height + PEATMAP + MODIS snow alb\n v11: NL3 + JPL veg height + PEATMAP + MODIS snow alb v2\n v12: NL3 + JPL veg height + PEATMAP + MODIS snow alb v2 + Argentina peatland fix \n - v14: v12 + Coupled MOM6/v2 ocean bathymetry (OM4) and v2 topography for atmosphere \n\n'''\ + v14: v12 + Coupled MOM6/v2 ocean bathymetry (OM4) and v2 topography for atmosphere \n + GM4: Ganymed-4_0\n\n'''\ +'''\ message_bc_other_in = ("Select BCs version of input restarts:\n" + message_bc_other) message_bc_other_new = ("Select BCs version for new restarts:\n" + message_bc_other) From 7d87929aa457f48fb4a02ab0e8213b51b07acb7c Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Fri, 24 Oct 2025 08:52:49 -0400 Subject: [PATCH 3/9] Fix up command line auto-split --- pre/remap_restart/remap_command_line.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py index bfa6a7c..189124f 100755 --- a/pre/remap_restart/remap_command_line.py +++ b/pre/remap_restart/remap_command_line.py @@ -102,14 +102,19 @@ def get_answers_from_command_line(cml): init_merra2(answers) elif cml.geosit: init_geosit(answers) - else: + else: answers["input:shared:bc_base"] = cml.in_bc_base answers["input:shared:omodel"] = cml.ocnmdlin answers["input:shared:bc_version"] = cml.bcvin + # If inputs are GM4, set split_saltwater to False which means split the saltwater restart + # as split_saltwater = True means the inputs are already split + answers["output:surface:split_saltwater"] = True + if 'GM4' in answers["input:shared:bc_version"]: + answers['output:surface:split_saltwater'] = False answers["input:surface:catch_model"]= cml.catch_model answers["input:shared:stretch"] = cml.in_stretch answers["input:shared:rst_dir"] = os.path.abspath(os.path.expanduser(cml.rst_dir)) - fvcore_info(answers) + fvcore_info(answers) ogrid = cml.oceanin if ogrid == "CS": ogrid = answers["input:shared:agrid"] From 89839a9478aa96800b76487e0158134c1253d093 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Fri, 24 Oct 2025 08:55:36 -0400 Subject: [PATCH 4/9] Fix bad python --- pre/remap_restart/remap_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 6dc82a8..77a2a2f 100755 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -96,7 +96,6 @@ v12: NL3 + JPL veg height + PEATMAP + MODIS snow alb v2 + Argentina peatland fix \n v14: v12 + Coupled MOM6/v2 ocean bathymetry (OM4) and v2 topography for atmosphere \n GM4: Ganymed-4_0\n\n'''\ -'''\ message_bc_other_in = ("Select BCs version of input restarts:\n" + message_bc_other) message_bc_other_new = ("Select BCs version for new restarts:\n" + message_bc_other) From f33f84745183d2256fe4bb14dc785129287081c3 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Fri, 24 Oct 2025 09:03:39 -0400 Subject: [PATCH 5/9] Different fix --- pre/remap_restart/remap_command_line.py | 5 ----- pre/remap_restart/remap_utils.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py index 189124f..0de66ba 100755 --- a/pre/remap_restart/remap_command_line.py +++ b/pre/remap_restart/remap_command_line.py @@ -106,11 +106,6 @@ def get_answers_from_command_line(cml): answers["input:shared:bc_base"] = cml.in_bc_base answers["input:shared:omodel"] = cml.ocnmdlin answers["input:shared:bc_version"] = cml.bcvin - # If inputs are GM4, set split_saltwater to False which means split the saltwater restart - # as split_saltwater = True means the inputs are already split - answers["output:surface:split_saltwater"] = True - if 'GM4' in answers["input:shared:bc_version"]: - answers['output:surface:split_saltwater'] = False answers["input:surface:catch_model"]= cml.catch_model answers["input:shared:stretch"] = cml.in_stretch answers["input:shared:rst_dir"] = os.path.abspath(os.path.expanduser(cml.rst_dir)) diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 77a2a2f..b385231 100755 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -586,7 +586,7 @@ def get_config_from_answers(answers, config_tpl = False): bc_version = config['output']['shared'].get('bc_version') config['output']['surface']['split_saltwater'] = True - if 'Ganymed' in bc_version : + if 'Ganymed' in bc_version or 'GM4' in bc_version: config['output']['surface']['split_saltwater'] = False return config From d10881525b6cd3945793c77ab8c0e068b05b8793 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Fri, 24 Oct 2025 09:47:35 -0400 Subject: [PATCH 6/9] Try removing salt --- .../remap_lake_landice_saltwater.py | 22 +++++++++++++++---- pre/remap_restart/remap_utils.py | 11 ++++++++-- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/pre/remap_restart/remap_lake_landice_saltwater.py b/pre/remap_restart/remap_lake_landice_saltwater.py index a687132..da20eec 100755 --- a/pre/remap_restart/remap_lake_landice_saltwater.py +++ b/pre/remap_restart/remap_lake_landice_saltwater.py @@ -138,7 +138,7 @@ def remap(self): zoom = config['input']['surface']['zoom'] if zoom is None : zoom = get_zoom(config) - + log_name = out_dir+'/remap_lake_landice_saltwater_log' if os.path.exists(log_name): os.remove(log_name) @@ -146,11 +146,11 @@ def remap(self): if (saltwater_internal): cmd = exe + out_til + ' ' + in_til + ' InData/'+ saltwater_internal + ' 0 ' + str(zoom) self.run_and_log(cmd, log_name) - + # split Saltwater Internal # NOTE: split_saltwater==True means that the input restarts are already split. # So we do not split them again. - if not config['output']['surface']['split_saltwater']: + if not config['input']['surface']['split_saltwater']: print("\nSplitting Saltwater Internal...\n") cmd = bindir+'/SaltIntSplitter.x ' + out_til + ' ' + 'OutData/' + saltwater_internal # subprocess.call(shlex.split(cmd)) @@ -158,6 +158,13 @@ def remap(self): seaice = '' self.run_and_log(cmd, log_name) + # We can now *remove* the unsplit saltwater internal restart to avoid confusion + unsplit_file = 'OutData/' + saltwater_internal + if os.path.exists(unsplit_file): + print('\n Removing unsplit saltwater internal restart: ' + unsplit_file + '\n') + os.remove(unsplit_file) + + if (saltwater_import): cmd = exe + out_til + ' ' + in_til + ' InData/'+ saltwater_import + ' 0 ' + str(zoom) self.run_and_log(cmd, log_name) @@ -165,7 +172,7 @@ def remap(self): # split Saltwater Import # NOTE: split_saltwater==True means that the input restarts are already split. # So we do not split them again. - if not config['output']['surface']['split_saltwater']: + if not config['input']['surface']['split_saltwater']: print("\nSplitting Saltwater Import...\n") cmd = bindir+'/SaltIntSplitter.x ' + out_til + ' ' + 'OutData/' + saltwater_import # subprocess.call(shlex.split(cmd)) @@ -173,6 +180,13 @@ def remap(self): seaice = '' self.run_and_log(cmd, log_name) + # We can now *remove* the unsplit saltwater import restart to avoid confusion + unsplit_file = 'OutData/' + saltwater_import + if os.path.exists(unsplit_file): + print('\n Removing unsplit saltwater import restart: ' + unsplit_file + '\n') + os.remove(unsplit_file) + + if (openwater): cmd = exe + out_til + ' ' + in_til + ' InData/' + openwater + ' 0 ' + str(zoom) self.run_and_log(cmd, log_name) diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index b385231..17be9ea 100755 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -584,9 +584,16 @@ def get_config_from_answers(answers, config_tpl = False): if len(keys) == 3: config[keys[0]][keys[1]][keys[2]] = value - bc_version = config['output']['shared'].get('bc_version') + # Do the inputs have a split saltwater? + input_bc_version = config['input']['shared'].get('bc_version') + config['input']['surface']['split_saltwater'] = True + if 'Ganymed' in input_bc_version or 'GM4' in input_bc_version: + config['input']['surface']['split_saltwater'] = False + + # Do the outputs need a split saltwater? + output_bc_version = config['output']['shared'].get('bc_version') config['output']['surface']['split_saltwater'] = True - if 'Ganymed' in bc_version or 'GM4' in bc_version: + if 'Ganymed' in output_bc_version or 'GM4' in output_bc_version: config['output']['surface']['split_saltwater'] = False return config From fcb42f2adcf2471b054550fba8724ed6d75d65a9 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Fri, 24 Oct 2025 10:29:18 -0400 Subject: [PATCH 7/9] Remove v13, make v12 default (for now) --- pre/remap_restart/remap_questions.py | 4 ++-- pre/remap_restart/remap_utils.py | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 23b3d5d..05ab180 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -292,7 +292,7 @@ def ask_questions(): "name": "output:shared:bc_version", "message": message_bc_ops_new, "choices": choices_bc_ops, - "default": "v13", + "default": "v12", "when": lambda x: x["input:shared:MERRA-2"] or x["input:shared:GEOS-IT"], }, @@ -301,7 +301,7 @@ def ask_questions(): "name": "output:shared:bc_version", "message": "Select BCs version for new restarts:\n", "choices": choices_bc_ops, - "default": "v13", + "default": "v12", "when": lambda x: not x["input:shared:MERRA-2"] and not x["input:shared:GEOS-IT"], }, diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 17be9ea..661e199 100755 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -30,9 +30,9 @@ # define "choices", "message" strings, and "validate" lists that are used multiple times # (and related definitions, even if they are used just once). -choices_bc_ops = ['v13', 'NL3', 'ICA', 'Other'] +choices_bc_ops = ['v12', 'NL3', 'ICA', 'Other'] -choices_bc_other = ['v06','v11','v12','v14','GM4'] +choices_bc_other = ['v06','v11','v14','GM4'] choices_bc_cmd = ['NL3', 'ICA', 'GM4', 'v06', 'v11','v12', 'v14'] @@ -80,7 +80,7 @@ message_bc_ops = f'''\n BCs version | ADAS tags | GCM tags typically used with BCs version -----------------|----------------------|----------------------------------------- - v13: v13 | future | 12.0 ... present + v12: v12 | future | 12.0 ... present NL3: Icarus-NLv3 | 5_25_1 ... present | Icarus_NL, 10.19 ... 11.7 ICA: Icarus | 5_17_0 ... 5_24_0_p1 | Icarus, Jason ... 10.18 ---------------------------------------------------------------------------------- @@ -89,11 +89,14 @@ message_bc_ops_in = ("Select boundary conditions (BCs) version of input restarts:\n" + message_bc_ops) message_bc_ops_new = ("Select boundary conditions (BCs) version for new restarts:\n" + message_bc_ops) +# v12 used to be in the "other" section but is now the default, but we preserve this information +# in a comment since v14 will probably soon be the new default and we'll want to move v12 back to "other". +#v12: NL3 + JPL veg height + PEATMAP + MODIS snow alb v2 + Argentina peatland fix \n + message_bc_other = f'''\n v06: NL3 + JPL veg height + PEATMAP + MODIS snow alb\n v11: NL3 + JPL veg height + PEATMAP + MODIS snow alb v2\n - v12: NL3 + JPL veg height + PEATMAP + MODIS snow alb v2 + Argentina peatland fix \n v14: v12 + Coupled MOM6/v2 ocean bathymetry (OM4) and v2 topography for atmosphere \n GM4: Ganymed-4_0\n\n'''\ From 41a3b52931d7814364b8b350b524573aba0f0db5 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Fri, 24 Oct 2025 10:31:17 -0400 Subject: [PATCH 8/9] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea0abd7..1312566 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added remapping for GEOS-IT restarts - Added new res C1120 - NOTE: If running on SLES15 remap tests will not be zero diff for GOCART RST but are zero diff for all other -- Add support for v13 BCs +- Add support for v14 BCs ### Changed From 765a968f36847fc3e054dc9f2bbcdc9d6061716a Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Fri, 24 Oct 2025 10:34:15 -0400 Subject: [PATCH 9/9] Do not remove salt import --- pre/remap_restart/remap_lake_landice_saltwater.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pre/remap_restart/remap_lake_landice_saltwater.py b/pre/remap_restart/remap_lake_landice_saltwater.py index da20eec..c502b1c 100755 --- a/pre/remap_restart/remap_lake_landice_saltwater.py +++ b/pre/remap_restart/remap_lake_landice_saltwater.py @@ -180,12 +180,6 @@ def remap(self): seaice = '' self.run_and_log(cmd, log_name) - # We can now *remove* the unsplit saltwater import restart to avoid confusion - unsplit_file = 'OutData/' + saltwater_import - if os.path.exists(unsplit_file): - print('\n Removing unsplit saltwater import restart: ' + unsplit_file + '\n') - os.remove(unsplit_file) - if (openwater): cmd = exe + out_til + ' ' + in_til + ' InData/' + openwater + ' 0 ' + str(zoom)