Skip to content

exclude tile2pfaf nc4 file which is not a tile file #160

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
weiyuan-jiang wants to merge 9 commits into
base: develop
Choose a base branch
from
Open

exclude tile2pfaf nc4 file which is not a tile file #160

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
68d08a1
exclude tile2pfaf nc4 file which is not a tile file
weiyuan-jiang Feb 11, 2026
97e8864
remove extra line in ldas.py
weiyuan-jiang Feb 11, 2026
0618cfc
fix bug to find extention and reorg. assert if domain is wrong
weiyuan-jiang Feb 18, 2026
5cae2f1
fix indent
weiyuan-jiang Feb 18, 2026
fc0e9ed
check bcs_path and resolution when RESTART = 1
weiyuan-jiang Mar 2, 2026
2e406ec
remove restart=0, add restart=3
weiyuan-jiang Mar 2, 2026
9ba7cac
rename new output file to "BCS_info.txt"
gmao-rreichle Mar 4, 2026
a65b644
edited comments about RESTART options (ldas.py, setup_utils.py)
gmao-rreichle Mar 4, 2026
2407a13
minimal edit of comment about RESTART (ldas.py)
gmao-rreichle Mar 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
163 changes: 64 additions & 99 deletions GEOSldas_App/ldas.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,18 +129,13 @@ def __init__(self, cmdLineArgs):
if 'RESTART' not in self.ExeInputs :
self.ExeInputs['RESTART'] = "1"

if self.ExeInputs['RESTART'].isdigit() :
if int(self.ExeInputs['RESTART']) ==0 :
self.ExeInputs['RESTART_ID'] = 'None'
self.ExeInputs['RESTART_DOMAIN'] = 'None'
self.ExeInputs['RESTART_PATH'] = 'None'
else:
if self.ExeInputs['RESTART'] =='G' :
self.ExeInputs['RESTART_DOMAIN'] = 'None'
else:
self.ExeInputs['RESTART_ID'] = 'None'
self.ExeInputs['RESTART_DOMAIN'] = 'None'
self.ExeInputs['RESTART_PATH'] = 'None'
RESTART_str = str(self.ExeInputs['RESTART'])
assert RESTART_str in ["1", "2", "3", "M"], "RESTART must be 1, 2, 3, or M"

if RESTART_str == 'M':
self.ExeInputs['RESTART_ID'] = 'None'
self.ExeInputs['RESTART_DOMAIN'] = 'None'
self.ExeInputs['RESTART_PATH'] = 'None'

### check if ldas is coupled to adas; if so, set/overwrite input parameters accordingly
if self.ladas_cpl > 0 :
Expand Down Expand Up @@ -266,7 +261,7 @@ def __init__(self, cmdLineArgs):
if self.ExeInputs['RESTART_PATH'][-1] != '/':
self.ExeInputs['RESTART_PATH'] = self.ExeInputs['RESTART_PATH']+'/'

# make sure catchment and vegdyn restart files ( at least one for each) exist
# make sure catchment and vegdyn restart files exist (at least one of each)
if 'CATCH_DEF_FILE' not in self.ExeInputs :
self.ExeInputs['CATCH_DEF_FILE']= self.bcs_dir_land + 'clsm/catchment.def'
if self.with_land :
Expand All @@ -279,15 +274,35 @@ def __init__(self, cmdLineArgs):
inpdir_ = self.bcs_dir_land
inpgeom_ = self.bcs_dir_geom

if self.ExeInputs['RESTART'] == '1' :
inp_ = self.ExeInputs['RESTART_PATH']+'/'.join([self.ExeInputs['RESTART_ID'],'output',

# find restart files and tile file in restart dir (if necessary)
inp_ = self.ExeInputs['RESTART_PATH']+'/'.join([self.ExeInputs['RESTART_ID'],'output',
self.ExeInputs['RESTART_DOMAIN'], 'rc_out/'])
if RESTART_str == '1' :
inpdir_ = inp_
inpgeom_ = inp_
BCS_txt = glob.glob(inp_ + 'BCS_info.txt')
if len(BCS_txt)== 0:
print("Warning: BCS_info.txt not found for restart experiment. User is responsible for ensuring consistency of restart and experiment BCS.")
elif len(BCS_txt)== 1:
BCS_tmp = parseInputFile(BCS_txt[0])
assert self.ExeInputs['BCS_PATH'] == BCS_tmp['BCS_PATH'], "BCS_PATH does not match path from restart dir (" + BCS_tmp['BCS_PATH'] + ")"
Copy link
Collaborator

@gmao-rreichle gmao-rreichle Mar 4, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@weiyuan-jiang : It's a bit more involved, but can we distinguish the VERSION from the path to the bcs? For offline simulations, the trailing dir name in the path is the bcs VERSION. It would be better to limit the check to just VERSION. This avoids triggering assert.false if the difference is only in the path (e.g., /discover/nobackup/projects vs /gpfsm/dnb05/projects.

assert self.ExeInputs['BCS_RESOLUTION'] == BCS_tmp['BCS_RESOLUTION'], "BCS_RESOLUTION does not match resolution from restart dir (" + BCS_tmp['BCS_RESOLUTION'] + ")"

txt_tile = glob.glob(inp_ + '*.domain')
if len(txt_tile) > 0:
domain_ = '.domain'
inpdir_ = inp_
inpgeom_ = inp_

elif RESTART_str == '2':
txt_tile = glob.glob(inp_ + '*.domain')
assert len(txt_tile) == 0, "For RESTART=2, must restart from (and run on) global domain"
in_tilefiles_ = glob.glob(inp_+'MAPL_*.til')
if len(in_tilefiles_) == 0 :
nc4_tmp = glob.glob(inp_+'/*.nc4')
in_tilesfile_ = [ item_ for item_ in nc4_tmp if 'tile2pfaf' not in item_ ]
if len(in_tilefiles_) == 0 :
in_tilefiles_ = glob.glob(inp_+'/*.til')
self.in_tilefile =os.path.realpath(in_tilefiles_[0])

inpdir_ = os.path.realpath(inpdir_)+'/'
inpgeom_ = os.path.realpath(inpgeom_)+'/'

Expand All @@ -296,7 +311,8 @@ def __init__(self, cmdLineArgs):
if 'MAPL_' in os.path.basename(f):
txt_tile = [f]
break
nc4_tile = glob.glob(inpgeom_ + '*.nc4' + domain_)
nc4_tmp = glob.glob(inpgeom_ + '*.nc4' + domain_)
nc4_tile = [ item_ for item_ in nc4_tmp if 'tile2pfaf' not in item_ ]
Copy link
Collaborator

@gmao-rreichle gmao-rreichle Mar 4, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@weiyuan-jiang : If we add an nc4 version of the cube-sphere-like tile file (instead of Yujin's "tile2pfaf" file, this bug fix no longer works. Probably the string 'tile2pfaf' would need to be replaced with 'Pfafstetter'.

if tile_file_format.upper() == 'TXT' : self.ExeInputs['TILING_FILE'] = txt_tile[0]
if tile_file_format.upper() == 'DEFAULT' : self.ExeInputs['TILING_FILE'] = (txt_tile+nc4_tile)[-1]

Expand All @@ -316,7 +332,7 @@ def __init__(self, cmdLineArgs):
tmptile = os.path.realpath(self.ExeInputs['TILING_FILE'])
extension = os.path.splitext(tmptile)[1]
if extension == '.domain':
extension = os.path.splitext(tmptile)[0]
extension = os.path.splitext(os.path.splitext(tmptile)[0])[1]
gridname_ =''
if extension == '.til':
gridname_ = linecache.getline(tmptile, 3).strip()
Expand Down Expand Up @@ -346,7 +362,8 @@ def __init__(self, cmdLineArgs):

for key,val in _domain_dic.items() :
if key in self.ExeInputs :
_domain_dic[key]= self.ExeInputs[key]
_domain_dic[key]= self.ExeInputs[key]
assert RESTART_str != 2, "Please comment out " + key + " event you set it global"
self.domain_def = tempfile.NamedTemporaryFile(mode='w', delete=False)
self.domain_def.write('&domain_inputs\n')
for key,val in _domain_dic.items() :
Expand All @@ -359,22 +376,7 @@ def __init__(self, cmdLineArgs):
self.domain_def.write('/\n')
self.domain_def.close()

# find restart files and tile files (if necessary)
RESTART_str = str(self.ExeInputs['RESTART'])

if RESTART_str == '2':
inpdir=self.ExeInputs['RESTART_PATH']+self.ExeInputs['RESTART_ID']+'/input/'
in_tilefiles_ = glob.glob(inpdir+'*tile.data')
if len(in_tilefiles_) == 0 :
inpdir=self.ExeInputs['RESTART_PATH']+self.ExeInputs['RESTART_ID']+'/output/'+self.ExeInputs['RESTART_DOMAIN']+'/rc_out/'
in_tilefiles_ = glob.glob(inpdir+'MAPL_*.til')
if len(in_tilefiles_) == 0 :
in_tilefiles_ = glob.glob(inpdir+'/*.til')
if len(in_tilefiles_) == 0 :
in_tilefiles_ = glob.glob(inpdir+'/*.nc4')
self.in_tilefile =os.path.realpath(in_tilefiles_[0])

if RESTART_str in ['1', '2']:
if RESTART_str in ['1','2','3']:
y4m2='Y%4d/M%02d' % (self.begDates[0].year, self.begDates[0].month)
y4m2d2_h2m2='%4d%02d%02d_%02d%02d' % (self.begDates[0].year, self.begDates[0].month,
self.begDates[0].day,self.begDates[0].hour,self.begDates[0].minute)
Expand All @@ -395,24 +397,6 @@ def __init__(self, cmdLineArgs):
landiceRstFile=self.in_rstdir+'/'+tmpFile
assert os.path.isfile(landiceRstFile), 'landice_internal_rst file [%s] does not exist!' %(landiceRstFile)

if RESTART_str == '0':
assert ( self.with_land and not self.with_landice), "RESTART = 0 is only for land"
if (self.catch == 'catch'):
self.in_rstdir = '/discover/nobackup/projects/gmao/ssd/land/l_data/LandRestarts_for_Regridding' \
'/Catch/M09/20170101/' #catch_internal_rst
self.in_tilefile = '/discover/nobackup/projects/gmao/ssd/land/l_data/geos5/bcs/CLSM_params' \
'/mkCatchParam_SMAP_L4SM_v002/SMAP_EASEv2_M09/SMAP_EASEv2_M09_3856x1624.til'
elif (self.catch == 'catchcnclm40'):
self.in_rstdir = '/discover/nobackup/projects/gmao/ssd/land/l_data/LandRestarts_for_Regridding' \
'/CatchCN/M36/20150301_0000/' #catchcnclm40_internal_dummy
self.in_tilefile = '/discover/nobackup/projects/gmao/bcs_shared/legacy_bcs/Heracles-NL/SMAP_EASEv2_M36/SMAP_EASEv2_M36_964x406.til'
elif (self.catch == 'catchcnclm45'):
self.in_rstdir = '/discover/nobackup/projects/gmao/ssd/land/l_data/LandRestarts_for_Regridding' \
'/CatchCN/M36/19800101_0000/' #catchcnclm45_internal_dummy
self.in_tilefile = '/discover/nobackup/projects/gmao/bcs_shared/legacy_bcs/Icarus-NLv3/Icarus-NLv3_EASE/SMAP_EASEv2_M36/SMAP_EASEv2_M36_964x406.til'
else:
sys.exit('need to provide at least dummy files')

# DEAL WITH mwRTM input from exec
self.assim = True if self.ExeInputs.get('LAND_ASSIM', 'NO').upper() == 'YES' and self.with_land else False
# verify mwrtm file
Expand All @@ -432,7 +416,7 @@ def __init__(self, cmdLineArgs):


# ------------------
# Read rm input file
# Read resource manager (rm) input file
# ------------------

if self.ladas_cpl == 0 :
Expand Down Expand Up @@ -472,7 +456,7 @@ def __init__(self, cmdLineArgs):
self.scratchdir = self.expdir + '/scratch'
self.blddirLn = self.expdir + '/build'
self.out_path = self.outdir + '/'+self.ExeInputs['EXP_DOMAIN']
self.bcsdir = self.outdir + '/'+self.ExeInputs['EXP_DOMAIN']+'/rc_out/'
self.rc_out = self.outdir + '/'+self.ExeInputs['EXP_DOMAIN']+'/rc_out/'
self.rstdir = self.outdir + '/'+self.ExeInputs['EXP_DOMAIN']+'/rs/'
self.exefyl = self.blddirLn + exefyl

Expand All @@ -493,8 +477,6 @@ def _verifyExeInputs(self):
ExeInputs = self.ExeInputs
#) verify keys
option = '1'
if (ExeInputs['RESTART'] == 'G' or ExeInputs['RESTART'] == '0'):
option = '0'

rqdExeInpKeys = getExeKeys(option)
for key in rqdExeInpKeys:
Expand Down Expand Up @@ -543,11 +525,6 @@ def _calculateJobSegments(self):
'%Y%m%d %H%M%S'
)
)
if self.ExeInputs['RESTART'].isdigit() :
if int(self.ExeInputs['RESTART']) == 0 :
print ("No restart file (cold restart): Forcing start date to January 1, 0z")
year = self.begDates[0].year
self.begDates[0]=datetime(year =year,month=1,day =1,hour =0, minute= 0,second= 0)

assert self.endDates[0]>self.begDates[0], \
'END_DATE <= BEG_DATE'
Expand Down Expand Up @@ -689,13 +666,13 @@ def createLnRstBc(self) :
# update tile file
tile= self.ExeInputs['TILING_FILE']
short_tile= os.path.basename(self.ExeInputs['TILING_FILE'])
newtile = self.bcsdir+'/'+short_tile
newtile = self.rc_out+'/'+short_tile
shutil.copy(tile, newtile)
tile=newtile
# if three extra lines exist, remove them and save it to inputdir

print ('\nCorrect the tile file if it is an old EASE tile format... \n')
EASEtile=self.bcsdir+'/MAPL_'+short_tile
EASEtile=self.rc_out+'/MAPL_'+short_tile
cmd = self.bindir + '/preprocess_ldas.x correctease '+ tile + ' '+ EASEtile
print ("cmd: " + cmd)

Expand Down Expand Up @@ -762,8 +739,8 @@ def createLnRstBc(self) :
bcs += [self.ExeInputs['VEGOPACITY_FILE']]
bcstmp=[]
for bcf in bcs :
shutil.copy(bcf, self.bcsdir+'/')
bcstmp=bcstmp+[self.bcsdir+'/'+os.path.basename(bcf)]
shutil.copy(bcf, self.rc_out+'/')
bcstmp=bcstmp+[self.rc_out+'/'+os.path.basename(bcf)]
bcs=bcstmp

if self.isZoomIn:
Expand Down Expand Up @@ -801,13 +778,6 @@ def createLnRstBc(self) :

myRstDir = self.inpdir + '/restart/'

rstpath = self.ExeInputs['RESTART_PATH']+ \
self.ExeInputs['RESTART_ID'] + \
'/output/'+self.ExeInputs['RESTART_DOMAIN']+'/rs/'
rcoutpath = self.ExeInputs['RESTART_PATH']+ \
self.ExeInputs['RESTART_ID'] + \
'/output/'+self.ExeInputs['RESTART_DOMAIN']+'/rc_out/'

# pass into remap_config_ldas
exp_id = self.ExeInputs['EXP_ID']
RESTART_str = str(self.ExeInputs['RESTART'])
Expand All @@ -816,6 +786,7 @@ def createLnRstBc(self) :
rstid = self.ExeInputs['RESTART_ID']
rstdomain = self.ExeInputs['RESTART_DOMAIN']
rstpath0 = self.ExeInputs['RESTART_PATH']
rstpath = rstpath0 + rstid + '/output/'+ rstdomain + '/rs/'

# just copy the landassim pert seed if it exists
for iens in range(self.nens) :
Expand All @@ -829,7 +800,7 @@ def createLnRstBc(self) :
self.has_landassim_seed = True
mk_outdir = self.exphome+'/'+exp_id+'/mk_restarts/'

if (RESTART_str != '1' and (self.with_land or self.with_landice)):
if (RESTART_str in ['2', 'M'] and (self.with_land or self.with_landice)):
bcs_path = self.ExeInputs['BCS_PATH']
while bcs_path[-1] == '/' : bcs_path = bcs_path[0:-1]
bc_base = os.path.dirname(bcs_path)
Expand All @@ -844,7 +815,7 @@ def createLnRstBc(self) :
config['input']['surface']['catch_tilefile'] = self.in_tilefile
config['input']['shared']['expid'] = self.ExeInputs['RESTART_ID']
config['input']['shared']['yyyymmddhh'] = YYYYMMDDHH
if RESTART_str != 'M':
if RESTART_str == '2':
config['input']['shared']['rst_dir'] = self.in_rstdir
config['input']['surface']['wemin'] = wemin_in
config['input']['surface']['catch_model'] = self.catch
Expand Down Expand Up @@ -911,24 +882,18 @@ def createLnRstBc(self) :
catchRstFile = ''
vegdynRstFile = ''
pertRstFile = ''
print ("restart: " + self.ExeInputs['RESTART'])
print ("restart: " + RESTART_str)

if self.with_land :
if self.ExeInputs['RESTART'].isdigit() :

if int(self.ExeInputs['RESTART']) == 0 or int(self.ExeInputs['RESTART']) == 2 :
if RESTART_str in ['2', 'M']: # remap_restart has generated the file in mk_restarts directory
vegdynRstFile = glob.glob(self.bcs_dir_land + 'vegdyn_*.dat')[0]
catchRstFile = glob.glob(self.exphome+'/'+exp_id+'/mk_restarts/*'+self.catch+'_internal_rst.'+YYYYMMDD+'*')[0]
if RESTART_str in ['1', '3']:
catchRstFile = rstpath+ensdir +'/'+ y4m2+'/'+self.ExeInputs['RESTART_ID']+'.'+self.catch+'_internal_rst.'+y4m2d2_h2m2
vegdynRstFile= rstpath+ensdir +'/'+self.ExeInputs['RESTART_ID']+ '.vegdyn_internal_rst'
if RESTART_str == '3':
vegdynRstFile = glob.glob(self.bcs_dir_land + 'vegdyn_*.dat')[0]
catchRstFile = glob.glob(self.exphome+'/'+exp_id+'/mk_restarts/*'+self.catch+'_internal_rst.'+YYYYMMDD+'*')[0]
else : # RESTART == 1
catchRstFile = rstpath+ensdir +'/'+ y4m2+'/'+self.ExeInputs['RESTART_ID']+'.'+self.catch+'_internal_rst.'+y4m2d2_h2m2
vegdynRstFile= rstpath+ensdir +'/'+self.ExeInputs['RESTART_ID']+ '.vegdyn_internal_rst'
if not os.path.isfile(vegdynRstFile): # no vegdyn restart from LDASsa
if not os.path.isfile(vegdynRstFile0):
vegdynRstFile = glob.glob(self.bcs_dir_land + 'vegdyn_*.dat')[0]
else :
vegdynRstFile = glob.glob(self.bcs_dir_land + 'vegdyn_*.dat')[0]
if self.with_land:
catchRstFile = glob.glob(self.exphome+'/'+exp_id+'/mk_restarts/*'+self.catch+'_internal_rst.'+YYYYMMDD+'*')[0]

# catchment restart file
if os.path.isfile(catchRstFile) :
Expand Down Expand Up @@ -968,11 +933,9 @@ def createLnRstBc(self) :

landiceRstFile = ''
if self.with_landice :
if RESTART_str == '0' :
exit("RESTART=0 not supported for landice tiles. Please use RESTART=M, 1, or 2")
if RESTART_str == '1' :
if RESTART_str in ['1', '3'] :
landiceRstFile = rstpath+ensdir +'/'+ y4m2+'/'+self.ExeInputs['RESTART_ID']+'.'+'landice_internal_rst.'+y4m2d2_h2m2
if RESTART_str == '2' or RESTART_str == 'M':
if RESTART_str in ['2', 'M']:
landiceRstFile = glob.glob(self.exphome+'/'+exp_id+'/mk_restarts/*'+'landice_internal_rst.'+YYYYMMDD+'*')[0]

if os.path.isfile(landiceRstFile) :
Expand Down Expand Up @@ -1010,13 +973,13 @@ def createLnRstBc(self) :
os.symlink(pertRstFile, myPertRst)

# catch_param restar file
catch_param_file = self.bcsdir+'/'+ y4m2+'/'+self.ExeInputs['EXP_ID']+'.ldas_catparam.'+y4m2d2_h2m2+'z.bin'
catch_param_file = self.rc_out+'/'+ y4m2+'/'+self.ExeInputs['EXP_ID']+'.ldas_catparam.'+y4m2d2_h2m2+'z.bin'
if self.with_land:
assert os.path.isfile(catch_param_file), "need catch_param file %s" % catch_param_file

if self.has_mwrtm :
mwRTMRstFile = self.mwrtm_file
mwRTMLocal = self.bcsdir+'/'+ y4m2+'/'+self.ExeInputs['EXP_ID']+'.ldas_mwRTMparam.'+y4m2d2_h2m2+'z.nc4'
mwRTMLocal = self.rc_out+'/'+ y4m2+'/'+self.ExeInputs['EXP_ID']+'.ldas_mwRTMparam.'+y4m2d2_h2m2+'z.nc4'
if self.isZoomIn :
print ("Creating the local mwRTM restart file... \n")
cmd= self.bindir +'/preprocess_ldas.x zoomin_mwrtmrst '+ mwRTMRstFile +' ' + mwRTMLocal + ' '+ tmp_f2g_file.name
Expand All @@ -1030,11 +993,13 @@ def createLnRstBc(self) :
mymwRTMRst = myRstDir+'/mwrtm_param_rst'
os.symlink(mwRTMRstFile, mymwRTMRst)

#create BCS_info.txt
with open(self.rc_out+'/BCS_info.txt','wt') as fout :
fout.write("BCS_PATH: " + self.ExeInputs['BCS_PATH'])
fout.write("BCS_RESOLUTION: " + self.ExeInputs['BCS_RESOLUTION'])
# update 'restart_path' to use relative path from outdir
print ("Updating restart path...")
self.ExeInputs['RESTART_PATH'] = myRstDir
#if os.path.isfile(tmp_f2g_file.name):
# os.remove(tmp_f2g_file.name)
status = True
return status

Expand Down
Loading
Loading