diff --git a/CHANGELOG.md b/CHANGELOG.md index a6df480..61776e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Update `remap_restarts.py` to first ask about remapping from GEOS-IT, then ask about MERRA-2. Add detection of MERRA-2 path as well + ### Fixed - Edited gcmpost.script to move up the location of $SOURCE/plot/.quickplotrc to restore missing plots from landscape.list for some users. diff --git a/pre/remap_restart/remap_analysis.py b/pre/remap_restart/remap_analysis.py index ddf85ae..3e9b8f8 100755 --- a/pre/remap_restart/remap_analysis.py +++ b/pre/remap_restart/remap_analysis.py @@ -17,7 +17,7 @@ import fileinput import ruamel.yaml from remap_base import remap_base -from remap_utils import get_label +from remap_utils import get_label, MERRA2_RST_BASE import fnmatch class analysis(remap_base): @@ -95,9 +95,9 @@ def remap(self): agrid_out = config['output']['shared']['agrid'] flags = "-g5 -res " + self.get_grid_kind(agrid_out.upper()) + " -nlevs " + str(nlevel) - dyn2dyn = fnmatch.filter(local_fs, '*bkg??_eta_rst*') + dyn2dyn = fnmatch.filter(local_fs, '*bkg??_eta_rst*') for f in dyn2dyn: - if "cbkg" not in f: + if "cbkg" not in f: f_orig = f + ".orig" shutil.move(f,f_orig) cmd = bindir + '/dyn2dyn.x ' + flags + ' -o ' + f + ' ' + f_orig @@ -106,15 +106,15 @@ def remap(self): for f in local_fs: fname = os.path.basename(f) - fname = fname + label + fname = fname + label shutil.move(f, out_dir+'/'+fname) # write lcv file - # (Info about "lcv" file provided by Ricardo Todling via email, 1 Sep 2023, + # (Info about "lcv" file provided by Ricardo Todling via email, 1 Sep 2023, # paraphrased by Rolf Reichle: # The lcv file is binary and inherits the nomenclature of the old GCM primary restarts [GEOS-4]. - # The file simply carries the date of the restarts, e.g., 20231201 210000. - # The file is required by the ADAS. - # LCV stands for Lagrangian Control Volume, which is what the grid coordinates of the model are + # The file simply carries the date of the restarts, e.g., 20231201 210000. + # The file is required by the ADAS. + # LCV stands for Lagrangian Control Volume, which is what the grid coordinates of the model are # [on the cubed-sphere for GEOS-5].) lcv = config['output']['analysis']['lcv'] if lcv : @@ -174,7 +174,7 @@ def copy_merra2(self): dd_ = yyyymmddhh_[6:8] hh_ = yyyymmddhh_[8:10] - merra_2_rst_dir = '/archive/users/gmao_ops/MERRA2/gmao_ops/GEOSadas-5_12_4/'+expid +'/rs/Y'+yyyy_ +'/M'+mm_+'/' + merra_2_rst_dir = MERRA2_RST_BASE + expid +'/rs/Y'+yyyy_ +'/M'+mm_+'/' rst_dir = self.config['input']['shared']['rst_dir'] + '/' os.makedirs(rst_dir, exist_ok = True) print(' Stage MERRA-2 analysis files \n from \n ' + merra_2_rst_dir + '\n to\n '+ rst_dir +'\n') @@ -182,7 +182,7 @@ def copy_merra2(self): rst_time = datetime(year=int(yyyy_), month=int(mm_), day=int(dd_), hour = int(hh_)) anafiles=[] - canafiles=[] + canafiles=[] for h in [3,4,5,6,7,8,9]: delt = timedelta(hours = h-3) new_time = rst_time + delt @@ -218,7 +218,7 @@ def copy_merra2(self): else: print('Warning: Cannot find '+f) - # satb files + # satb files for ftype in ['satbias', 'satbang']: fname = expid + '.ana_'+ftype+'_rst.'+ymd+'_'+newhh+'z.txt' f = m2_rst_dir+'/'+fname diff --git a/pre/remap_restart/remap_base.py b/pre/remap_restart/remap_base.py index 70de50e..df04c62 100755 --- a/pre/remap_restart/remap_base.py +++ b/pre/remap_restart/remap_base.py @@ -40,7 +40,7 @@ def remove_geosit(self): def copy_without_remap(self, restarts_in, compared_file_in, compared_file_out, suffix, catch=False): # -# Determine if remapping is needed for a group of restart files, or if the input restart files +# Determine if remapping is needed for a group of restart files, or if the input restart files # can just simply be copied to the output dir, based on the following dependency table: # # restarts | agrid/stretch | #levels | topo files | tile file | bcs version @@ -96,21 +96,21 @@ def copy_without_remap(self, restarts_in, compared_file_in, compared_file_out, s def copy_geosit(self): if not self.config['input']['shared']['GEOS-IT']: - return - + return + expid = self.config['input']['shared']['expid'] yyyymmddhh_ = str(self.config['input']['shared']['yyyymmddhh']) yyyy_ = yyyymmddhh_[0:4] mm_ = yyyymmddhh_[4:6] day_ = yyyymmddhh_[6:8] # Extract the day from yyyymmddhh_ - - time_suffix = '_21z' - time_suffix_nc4 = '_2100z' - - geos_it_rst_dir = '/discover/nobackup/projects/gmao/geos-it/dao_ops/archive/' + expid + '/rs/Y' + yyyy_ + '/M' + mm_ + '/' + + time_suffix = '_21z' + time_suffix_nc4 = '_2100z' + + geos_it_rst_dir = GEOSIT_RST_BASE + expid + '/rs/Y' + yyyy_ + '/M' + mm_ + '/' rst_dir = self.config['input']['shared']['rst_dir'] + '/' os.makedirs(rst_dir, exist_ok=True) - + print('Stage GEOS-IT restarts \n from \n ' + geos_it_rst_dir + '\n to\n ' + rst_dir + '\n') # Only use the specific day from yyyymmddhh_ diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py index de29e1e..aca38c0 100755 --- a/pre/remap_restart/remap_catchANDcn.py +++ b/pre/remap_restart/remap_catchANDcn.py @@ -146,6 +146,7 @@ def remap(self): CONSTRAINT = '"[cas|mil]"' account = config['slurm_pbs']['account'] + # even if the (MERRA-2) input restarts are binary, the output restarts will always be nc4 (remap_bin2nc.py) suffix = '_rst.' + suffix out_rstfile = expid + os.path.basename(in_rstfile).split('_rst')[0].split('.')[-1]+suffix @@ -241,7 +242,7 @@ def copy_merra2(self): hh_ = yyyymmddhh_[8:10] suffix = yyyymmddhh_[0:8]+'_'+ hh_ + 'z.bin' - merra_2_rst_dir = '/archive/users/gmao_ops/MERRA2/gmao_ops/GEOSadas-5_12_4/'+expid +'/rs/Y'+yyyy_ +'/M'+mm_+'/' + merra_2_rst_dir = MERRA2_RST_BASE + expid +'/rs/Y'+yyyy_ +'/M'+mm_+'/' rst_dir = self.config['input']['shared']['rst_dir'] + '/' os.makedirs(rst_dir, exist_ok = True) print(' Copy MERRA-2 catchment Restart \n from \n ' + merra_2_rst_dir + '\n to\n '+ rst_dir +'\n') diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py index 9fd87b2..c893980 100755 --- a/pre/remap_restart/remap_command_line.py +++ b/pre/remap_restart/remap_command_line.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # # remap_restarts package: -# remap_command_line.py parses and converts the command-line arguments and converts the information +# remap_command_line.py parses and converts the command-line arguments and converts the information # into matching "Answers" for the questionary (remap_questions.py) # import os @@ -64,7 +64,7 @@ def parse_args(program_description): p_command.add_argument('-ocnmdlout', default='data', help='Ocean model for new restarts', choices=choices_omodel) p_command.add_argument('-in_stretch', default=False, help='Stretched CS params of input restarts', choices=choices_stretch) p_command.add_argument('-out_stretch', default=False, help='Stretched CS params for new restarts', choices=choices_stretch) - + # Unlike remap_questions.py, command-line feature does not deduce Catch vs. CatchCN[40,51] for simplicity, thus requires input argument p_command.add_argument('-catch_model',default='catch', help='Catchment[CN] model', choices=choices_catchmodel) @@ -99,14 +99,14 @@ def get_answers_from_command_line(cml): init_merra2(answers) elif cml.geosit: init_geosit(answers) - else: + else: answers["input:shared:bc_base"] = cml.in_bc_base answers["input:shared:omodel"] = cml.ocnmdlin answers["input:shared:bc_version"] = cml.bcvin answers["input:surface:catch_model"]= cml.catch_model answers["input:shared:stretch"] = cml.in_stretch answers["input:shared:rst_dir"] = os.path.abspath(os.path.expanduser(cml.rst_dir)) - fvcore_info(answers) + fvcore_info(answers) ogrid = cml.oceanin if ogrid == "CS": ogrid = answers["input:shared:agrid"] @@ -143,7 +143,7 @@ def get_answers_from_command_line(cml): answers["output:air:agcm_import_rst"] = not cml.noagcm_import_rst - if cml.zoom: + if cml.zoom: answers["input:surface:zoom"] = cml.zoom else: answers["input:surface:zoom"] = get_zoom(answers) @@ -161,7 +161,7 @@ def get_answers_from_command_line(cml): answers["slurm_pbs:account"] = cml.account answers["slurm_pbs:qos"] = cml.qos answers["slurm_pbs:partition"] = cml.partition - + return answers if __name__ == "__main__": @@ -174,7 +174,7 @@ def get_answers_from_command_line(cml): with open("raw_command.yaml", "w") as f: yaml.dump(config, f) - config = get_config_from_answers(answers, config_tpl= True) + config = get_config_from_answers(answers, config_tpl= True) with open("params_from_command.yaml", "w") as f: yaml.dump(config, f) diff --git a/pre/remap_restart/remap_lake_landice_saltwater.py b/pre/remap_restart/remap_lake_landice_saltwater.py index ebbd117..8ab48d0 100755 --- a/pre/remap_restart/remap_lake_landice_saltwater.py +++ b/pre/remap_restart/remap_lake_landice_saltwater.py @@ -17,7 +17,7 @@ import ruamel.yaml import shlex from remap_base import remap_base -from remap_utils import get_label, get_geomdir, get_zoom +from remap_utils import get_label, get_geomdir, get_zoom, MERRA2_RST_BASE from remap_bin2nc import bin2nc class lake_landice_saltwater(remap_base): @@ -138,7 +138,7 @@ def remap(self): zoom = config['input']['surface']['zoom'] if zoom is None : zoom = get_zoom(config) - + log_name = out_dir+'/remap_lake_landice_saltwater_log' if os.path.exists(log_name): os.remove(log_name) @@ -264,7 +264,7 @@ def copy_merra2(self): hh_ = yyyymmddhh_[8:10] suffix = yyyymmddhh_[0:8]+'_'+ hh_ + 'z.bin' - merra_2_rst_dir = '/archive/users/gmao_ops/MERRA2/gmao_ops/GEOSadas-5_12_4/'+expid +'/rs/Y'+yyyy_ +'/M'+mm_+'/' + merra_2_rst_dir = MERRA2_RST_BASE + expid +'/rs/Y'+yyyy_ +'/M'+mm_+'/' rst_dir = self.config['input']['shared']['rst_dir'] + '/' os.makedirs(rst_dir, exist_ok = True) print(' Copy MERRA-2 surface restarts \n from \n ' + merra_2_rst_dir + '\n to\n '+ rst_dir +'\n') diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 5ec3bc0..759b219 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -1,17 +1,13 @@ #!/usr/bin/env python3 # # remap_restarts package: -# interactive questionary to create a yaml configuration file (remap_params.yaml) and +# interactive questionary to create a yaml configuration file (remap_params.yaml) and # a matching command line argument string (remap_restarts.CMD) # # import os -import subprocess -import shlex import ruamel.yaml -import shutil import questionary -import glob from remap_utils import * def echo_level(x): @@ -36,10 +32,10 @@ def echo_bcs(x,opt): return False def default_partition(x): - if x['slurm_pbs:qos'] == 'debug': - x['slurm_pbs:partition'] = 'compute' - return False - return True + if x['slurm_pbs:qos'] == 'debug': + x['slurm_pbs:partition'] = 'compute' + return False + return True def validate_merra2_time(text): if len(text) == 10 : @@ -47,7 +43,7 @@ def validate_merra2_time(text): if hh in ['03','09','15','21']: return True else: - return False + return False else: return False @@ -72,7 +68,7 @@ def SITE_GEOSIT(x): if GEOS_SITE == "NAS": x['input:shared:GEOS-IT']= False return False - return True + return True def ask_questions(): @@ -82,44 +78,44 @@ def ask_questions(): questions = [ { "type": "confirm", - "name": "input:shared:MERRA-2", - "message": "Remap archived MERRA-2 restarts? (Only if NCCS/Discover directory /archive is available; else, select 'N' and complete full config; requires nc4 restarts.)\n", + "name": "input:shared:GEOS-IT", + "message": "Remap from archived GEOS-IT restarts?\n", "default": False, - "when": lambda x: SITE_MERRA2(x), + "when": lambda x: SITE_GEOSIT(x), }, { "type": "confirm", - "name": "input:shared:GEOS-IT", - "message": "Remap archived GEOS-IT restarts? (NCCS/Discover only; elsewhere, select 'N' and complete full config; requires nc4 restarts.)\n", + "name": "input:shared:MERRA-2", + "message": "Remap from archived MERRA-2 restarts?\n", "default": False, - "when": lambda x: SITE_GEOSIT(x) and not x.get("input:shared:MERRA-2", False), + "when": lambda x: SITE_MERRA2(x) and not x.get("input:shared:GEOS-IT") and os.path.isdir(MERRA2_RST_BASE), }, { "type": "path", "name": "input:shared:rst_dir", "message": "Enter input directory with restart files to be remapped:\n", - "when": lambda x: not x.get("input:shared:MERRA-2", False) and not x.get("input:shared:GEOS-IT", False), + "when": lambda x: not x.get("input:shared:MERRA-2") and not x.get("input:shared:GEOS-IT"), }, { "type": "text", "name": "input:shared:yyyymmddhh", "message": "Enter the restart date and hour (YYYYMMDDHH):\n", "validate": lambda text: len(text) == 10, - "when": lambda x: not x.get('input:shared:MERRA-2', False) and not x.get('input:shared:GEOS-IT', False) and not fvcore_info(x), + "when": lambda x: not x.get('input:shared:MERRA-2') and not x.get('input:shared:GEOS-IT') and not fvcore_info(x), }, { "type": "text", "name": "input:shared:yyyymmddhh", "message": "Enter the restart date and hour (YYYYMMDDHH, hour = 03, 09, 15, or 21 [z]):\n", "validate": lambda text: validate_merra2_time(text), - "when": lambda x: x.get("input:shared:MERRA-2", False), + "when": lambda x: x.get("input:shared:MERRA-2"), }, { "type": "text", "name": "input:shared:yyyymmddhh", "message": "Enter the restart date and hour (YYYYMMDDHH, Only days available are 14th and 28th and only hour = 21 [z]):\n", "validate": lambda text: validate_geosit_time(text), - "when": lambda x: x.get("input:shared:GEOS-IT", False) and not x.get("input:shared:MERRA-2", False), + "when": lambda x: x.get("input:shared:GEOS-IT") and not x.get("input:shared:MERRA-2"), }, { @@ -152,25 +148,25 @@ def ask_questions(): "name": "input:shared:agrid", "message": message_agrid_in, "validate": lambda text: text in validate_agrid, - "when": lambda x: not x['input:shared:MERRA-2'] and not x['input:shared:GEOS-IT'] and not fvcore_info(x), + "when": lambda x: not x.get('input:shared:MERRA-2') and not x.get('input:shared:GEOS-IT') and not fvcore_info(x), }, - { + { "type": "select", "name": "input:shared:omodel", "message": "Select ocean model of input restarts:\n", "choices": choices_omodel, "default": "data", - "when": lambda x: not x['input:shared:MERRA-2'] and not x['input:shared:GEOS-IT'], + "when": lambda x: not x.get('input:shared:MERRA-2') and not x.get('input:shared:GEOS-IT'), }, - { + { "type": "select", "name": "input:shared:ogrid", "message": message_ogrid_in, "choices": choices_ogrid_data, "default": lambda x: data_ocean_default(x.get('input:shared:agrid')), - "when": lambda x: x.get('input:shared:omodel') == 'data' and not x['input:shared:MERRA-2'] and not x['input:shared:GEOS-IT'], + "when": lambda x: x.get('input:shared:omodel') == 'data' and not x.get('input:shared:MERRA-2') and not x.get('input:shared:GEOS-IT'), }, # dummy (invisible) question to remove parenthetical comments from selected input:shared:ogrid @@ -208,7 +204,7 @@ def ask_questions(): { "type": "select", "name": "output:shared:stretch", - "message": message_stretch, + "message": message_stretch, "choices": choices_stretch[1:3], "when": lambda x : x['output:shared:stretch'], }, @@ -217,7 +213,7 @@ def ask_questions(): "type": "select", "name": "output:shared:agrid", "message": "Select resolution of SG001 grid for new restarts: \n", - "choices": choices_res_SG001, + "choices": choices_res_SG001, "when": lambda x : x.get('output:shared:stretch') == 'SG001', }, @@ -225,7 +221,7 @@ def ask_questions(): "type": "select", "name": "output:shared:agrid", "message": "Select resolution of SG002 grid for new restarts: \n", - "choices": choices_res_SG002, + "choices": choices_res_SG002, "when": lambda x : x.get('output:shared:stretch') == 'SG002', }, @@ -274,7 +270,7 @@ def ask_questions(): "name": "input:shared:bc_version", "message": message_bc_ops_in, "choices": choices_bc_ops, - "when": lambda x: not x["input:shared:MERRA-2"] and not x["input:shared:GEOS-IT"], + "when": lambda x: not x.get("input:shared:MERRA-2") and not x.get("input:shared:GEOS-IT"), }, { @@ -291,7 +287,7 @@ def ask_questions(): "message": message_bc_ops_new, "choices": choices_bc_ops, "default": "NL3", - "when": lambda x: x["input:shared:MERRA-2"] or x["input:shared:GEOS-IT"], + "when": lambda x: x.get("input:shared:MERRA-2") or x.get("input:shared:GEOS-IT"), }, { @@ -300,7 +296,7 @@ def ask_questions(): "message": "Select BCs version for new restarts:\n", "choices": choices_bc_ops, "default": "NL3", - "when": lambda x: not x["input:shared:MERRA-2"] and not x["input:shared:GEOS-IT"], + "when": lambda x: not x.get("input:shared:MERRA-2") and not x.get("input:shared:GEOS-IT"), }, { @@ -376,8 +372,8 @@ def ask_questions(): { "type": "confirm", "name": "output:air:agcm_import_rst", - "message": f'''Remap agcm_import_rst (a.k.a. IAU) file needed for REPLAY runs? - (NOTE: Preferred method is to regenerate IAU file, + "message": f'''Remap agcm_import_rst (a.k.a. IAU) file needed for REPLAY runs? + (NOTE: Preferred method is to regenerate IAU file, but IF requested, remapping will be performed.)''', "default": False, "when": lambda x: echo_level(x), @@ -459,14 +455,15 @@ def ask_questions(): answers['input:shared:rst_dir'] = os.path.abspath(os.path.expanduser(answers['input:shared:rst_dir'])) answers['output:shared:out_dir'] = os.path.abspath(os.path.expanduser(answers['output:shared:out_dir'])) - if answers.get('input:air:nlevel') : del answers['input:air:nlevel'] - if answers["output:surface:remap"] and not answers["input:shared:MERRA-2"] and not answers["input:shared:GEOS-IT"]: + if answers.get('input:air:nlevel'): + del answers['input:air:nlevel'] + if answers["output:surface:remap"] and not answers.get("input:shared:MERRA-2") and not answers.get("input:shared:GEOS-IT"): answers["input:surface:catch_model"] = catch_model(answers) answers["output:surface:remap_water"] = answers["output:surface:remap"] answers["output:surface:remap_catch"] = answers["output:surface:remap"] del answers["output:surface:remap"] - + return answers if __name__ == "__main__": diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index 6436c07..f7cdab0 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -370,7 +370,7 @@ def copy_merra2(self): hh_ = yyyymmddhh_[8:10] suffix = yyyymmddhh_[0:8]+'_'+ hh_ + 'z.bin' - merra_2_rst_dir = '/archive/users/gmao_ops/MERRA2/gmao_ops/GEOSadas-5_12_4/'+expid +'/rs/Y'+yyyy_ +'/M'+mm_+'/' + merra_2_rst_dir = MERRA2_RST_BASE + expid +'/rs/Y'+yyyy_ +'/M'+mm_+'/' rst_dir = self.config['input']['shared']['rst_dir'] + '/' os.makedirs(rst_dir, exist_ok = True) print(' Stage MERRA-2 upper air restarts \n from \n ' + merra_2_rst_dir + '\n to\n '+ rst_dir +'\n') diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 783302a..467fd80 100755 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -45,6 +45,10 @@ choices_ogrid_cmd = ['360x180', '1440x720', '2880x1440', 'CS'] + choices_ogrid_cpld +# Base locations of MERRA2 and GEOS-IT Restarts +MERRA2_RST_BASE = '/archive/users/gmao_ops/MERRA2/gmao_ops/GEOSadas-5_12_4/' +GEOSIT_RST_BASE = '/discover/nobackup/projects/gmao/geos-it/dao_ops/archive/' + # the following needs more cleanup; e.g., first define list of SGxxx names and parameters (i.e., STRETCH_GRID), # then assemble message_stretch and choices_stretch using this definition @@ -331,10 +335,10 @@ def wemin_default(bc_version): return default_ def show_wemin_default(x): - if x['input:shared:MERRA-2']: + if x.get('input:shared:MERRA-2'): x['input:surface:wemin'] = '26' return False - elif x['input:shared:GEOS-IT']: + elif x.get('input:shared:GEOS-IT'): x['input:surface:wemin'] = '13' return False else: @@ -342,7 +346,7 @@ def show_wemin_default(x): return True def get_zoom(x): - # "zoom" approximates the (integer) number of grid cells per degree lat or lon (min=1, max=8); + # "zoom" approximates the (integer) number of grid cells per degree lat or lon (min=1, max=8); # for EASEv2 grid and lat/lon grid, always use the default value of 8. zoom_ = '8' if x.get('input:shared:MERRA-2') or x.get('input:shared:GEOS-IT'): @@ -618,6 +622,7 @@ def get_default_bc_base(): return choices_bc_base[0] return choices_bc_base[1] + def get_topodir(bc_base, bc_version, agrid=None, ogrid=None, omodel=None, stretch=None): gridStr = get_resolutions(agrid=agrid, ogrid=ogrid, omodel=omodel,stretch=stretch) agrid_name = gridStr.split('_')[0]