scm_setup

#!/usr/bin/env bash

SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
  DIR="$( builtin cd -P "$( dirname "$SOURCE" )" && pwd )"
  SOURCE="$(readlink "$SOURCE")"
  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
SCRIPTDIR="$( builtin cd -P "$( dirname "$SOURCE" )" && pwd )"

INSTALLDIR=$(dirname $SCRIPTDIR)

if [[ ! -e $INSTALLDIR/bin/GEOSgcm.x ]]
then
   echo "ERROR: $INSTALLDIR/bin/GEOSgcm.x does not exist"
   echo "You must execute $0 from the installation directory"
   exit 1
fi

CURRDIR=$(pwd)

# -------------
# Allowed cases
# -------------

# see geos5 wiki for information about these experiments

allowed_cases=()
allowed_cases+=("arm_97jul")
allowed_cases+=("merra_arm97jul")
allowed_cases+=("barrow2013")
allowed_cases+=("merra-Kan-L")
allowed_cases+=("arm_kwjx")
allowed_cases+=("merra_kwjx")
allowed_cases+=("COARE")
allowed_cases+=("merra_coare")
allowed_cases+=("TRMM_LBA")
allowed_cases+=("merra_trmm")
allowed_cases+=("arm_scsmex")
allowed_cases+=("merra_arm_scmx")
allowed_cases+=("merra_armsgp")
allowed_cases+=("scm_cfmip-p6")
allowed_cases+=("scm_cfmip-p11")
allowed_cases+=("scm_cfmip-p12")
allowed_cases+=("scm_cfmip-s6")
allowed_cases+=("scm_cfmip-s11")
allowed_cases+=("scm_cfmip-s12")
allowed_cases+=("scm_cfmip2-p6")
allowed_cases+=("scm_cfmip2-p11")
allowed_cases+=("scm_cfmip2-p12")
allowed_cases+=("scm_cfmip2-s6")
allowed_cases+=("scm_cfmip2-s11")
allowed_cases+=("scm_cfmip2-s12")
allowed_cases+=("merra_twp")
allowed_cases+=("armtwp_ice")
allowed_cases+=("merra_namma")
allowed_cases+=("merra_namecore")
allowed_cases+=("merra_nameaznm")
allowed_cases+=("e572p1_discaq")
allowed_cases+=("merra2_arise_ice_2014")
allowed_cases+=("merra2_arise_opnw_2014")
allowed_cases+=("merra2_ascos2008")
allowed_cases+=("merra2_barrow2013")
allowed_cases+=("LASIC_aug2016")
allowed_cases+=("bomex")
allowed_cases+=("dycoms_rf01")
allowed_cases+=("arm_97jun")
allowed_cases+=("dry_cbl")

sorted_cases=($(for each in ${allowed_cases[@]}; do echo $each; done | sort))

# -----
# Usage
# -----

usage ()
{
   echo "Usage: $0 --expdir <directory> --case <case> --account <account> (--num_levels <# of levels>)"
   echo
   echo "  Required arguments: "
   echo "    --expdir: experiment directory"
   echo
   echo "    --case: case to run where the allowed cases are:"
   for i in ${!sorted_cases[@]}
   do
      printf "       %-25s" "${sorted_cases[$i]}"
      if (( $i%3 == 2 ))
      then
         printf "\n"
      fi
   done
   echo
   echo "    --account: account under which to run the job"
   echo
   echo "  Optional argument: "
   echo "    --num_levels: number of levels (72, 71, 91, 132, 137, and 181 allowed)."
   echo
   echo "  Optional env vars: "
   echo "    When running on the TBC data, you can set TINYBCS_PATH to point to the dataset or you can answer the prompt."
   echo
   echo
}

# ------------
# Set defaults
# ------------
selected_case=NULL
expdir=NULL
account=NULL
nlev=72

while [ "${1+defined}" ]
do
   case "$1" in
      "--case")
         shift
         selected_case=$1
         shift
         ;;
      "--expdir")
         shift
         expdir=$1
         shift
         ;;
      "--account")
         shift
         account=$1
         shift
         ;;
      "--num_levels")
         shift
         nlev=$1
         shift
         ;;
      "--help" | "-h")
         usage
         exit 0
         ;;
      *)
         echo "Unknown option: $1"
         echo ""
         usage
         exit 1
         ;;
   esac
done

if [[ "$selected_case" == "NULL" ]]
then
   echo "ERROR: Case argument was not passed in"
   echo
   usage
   exit 1
fi

if [[ "$expdir" == "NULL" ]]
then
   echo "ERROR: Experiment directory was not passed in"
   echo
   usage
   exit 1
fi

if [[ "$account" == "NULL" ]]
then
   echo "ERROR: Account was not passed in"
   echo
   usage
   exit 1
fi

if [[ -d $expdir ]]
then
   echo "ERROR: $expdir exists! Please select different directory"
   exit 1
fi

if [[ $nlev -ne 72 ]]  &&  [[ $nlev -ne 132 ]] && [[ $nlev -ne 91 ]] && [[ $nlev -ne 137 ]] && [[ $nlev -ne 181 ]] && [[ $nlev -ne 71 ]]
then
   echo "ERROR: nlev needs to be 72, 71, 91, 132, 137, or 181"
   usage
   exit 1
fi

containsElement () {
  local e
  match="$1"
  #echo "Looking for $match"
  shift
  for e
  do
     #echo "Testing $match against $e"
     [[ "$e" == "$match" ]] && return 0
  done
  return 1
}

if ! containsElement "$selected_case" "${allowed_cases[@]}"
then
   echo "ERROR! Requested case $selected_case is not an allowed case"
   usage
   exit 1
fi

echo "Using case: $selected_case"
echo "Number of levels: $nlev"
echo "Experiment directory: $expdir"
echo "Account: $account"
echo

# -----------------
# Detect usual bits
# -----------------

ARCH=$(uname -s)
MACH=$(uname -m)
NODE=$(uname -n)

# ------------------------------
# Define an in-place sed command
# Because Mac sed is stupid old,
# use gsed if found.
# ------------------------------

if [[ $ARCH == Darwin ]]
then
   if [[ $(command -v gsed) ]]
   then
      echo "Found gsed on macOS. You are smart!"
      SED="$(command -v gsed) "
      ISED="$SED -i "
   else
      echo "It is recommended to use GNU sed since macOS default"
      echo "sed is a useless BSD variant. Consider installing"
      echo "GNU sed from a packager like Homebrew."
      SED="$(command -v sed) "
      ISED="$SED -i.macbak "
   fi
   PRELOAD_COMMAND='DYLD_INSERT_LIBRARIES'
   LD_LIBRARY_PATH_CMD='DYLD_LIBRARY_PATH'
   # On macOS we seem to need to call mpirun directly and not use esma_mpirun
   # For some reason SIP does not let the libraries be preloaded
   RUN_CMD='mpirun -np '
   FILE_WEIGHTS='false'
else
   SED="$(command -v sed) "
   ISED="$SED -i "
   PRELOAD_COMMAND='LD_PRELOAD'
   LD_LIBRARY_PATH_CMD='LD_LIBRARY_PATH'
   RUN_CMD='$GEOSBIN/esma_mpirun -np '
   FILE_WEIGHTS='true'
fi

case $selected_case in
   "arm_97jul")
      DATFILE="arm_97jul.dat"
      ;;
   "arm_97jun")
      DATFILE="arm_97jul.dat"
      ;;
   "bomex")
      DATFILE="bomex.dat"
      ;;
   "dycoms_rf01")
      DATFILE="dycoms_rf01.dat"
      ;;
   "dry_cbl")
      DATFILE="dcbl.dat"
      ;;
   "merra_arm97jul")
      DATFILE="merra_arm97jul.dat"
      ;;
   "barrow2013")
      DATFILE="barrow2013.dat"
      ;;
   "merra-Kan-L")
      DATFILE="merra-Kan-L.dat"
      ;;
   "arm_kwjx")
      DATFILE="arm_kwjx.dat"
      ;;
   "merra_kwjx")
      DATFILE="merra_kwjx.dat"
      ;;
   "merra_coare")
      DATFILE="merra_coare.dat"
      ;;
   "COARE")
      DATFILE="TOGA_COARE.dat"
      ;;
   "TRMM_LBA")
      DATFILE="TRMM_LBA.dat"
      ;;
   "merra_trmm")
      DATFILE="merra_trmm.dat"
      ;;
   "arm_scsmex")
      DATFILE="arm_scmx.dat"
      ;;
   "merra_arm_scmx")
      DATFILE="merra_scmx.dat"
      ;;
   "merra_armsgp")
      DATFILE="merra_armsgp.dat"
      ;;
   "scm_cfmip-p6")
      DATFILE="cfmip_p6.bin"
      ;;
   "scm_cfmip-p11")
      DATFILE="cfmip_p11.bin"
      ;;
   "scm_cfmip-p12")
      DATFILE="cfmip_p12.bin"
      ;;
   "scm_cfmip-s6")
      DATFILE="cfmip_s6.bin"
      ;;
   "scm_cfmip-s11")
      DATFILE="cfmip_s11.bin"
      ;;
   "scm_cfmip-s12")
      DATFILE="cfmip_s12.bin"
      ;;
   "scm_cfmip2-p6")
      DATFILE="cfmip2_p6.bin"
      ;;
   "scm_cfmip2-p11")
      DATFILE="cfmip2_p11.bin"
      ;;
   "scm_cfmip2-p12")
      DATFILE="cfmip2_p12.bin"
      ;;
   "scm_cfmip2-s6")
      DATFILE="cfmip2_s6.bin"
      ;;
   "scm_cfmip2-s11")
      DATFILE="cfmip2_s11.bin"
      ;;
   "scm_cfmip2-s12")
      DATFILE="cfmip2_s12.bin"
      ;;
   "armtwp_ice")
      DATFILE="armtwp_ice.dat"
      ;;
   "merra_twp")
      DATFILE="merra_twp.dat"
      ;;
   "merra_namma")
      DATFILE="merra_namma.dat"
      ;;
   "merra_namecore")
      DATFILE="merra_namecore.dat"
      ;;
   "merra_nameaznm")
      DATFILE="merra_nameaznm.dat"
      ;;
   "e572p1_discaq")
      DATFILE="e572p1_discaq.dat"
      ;;
   "merra2_arise_ice_2014")
      DATFILE="merra2_arise_ice_2014.dat"
      ;;
   "merra2_arise_opnw_2014")
      DATFILE="merra2_arise_opnw_2014.dat"
      ;;
   "merra2_ascos2008")
      DATFILE="merra2_ascos2008.dat"
      ;;
   "merra2_barrow2013")
      DATFILE="barrow2013.dat"
      ;;
   "LASIC_aug2016")
      DATFILE="merra2_lasic_aug2016.dat"
      ;;
   *)
      echo "You should not be here!"
      echo "If you are here, there is a case in allowed_cases"
      echo "that does not have a corresponding DATFILE"
      exit 9
      ;;
esac

SITE=$(awk '{print $2}' $INSTALLDIR/etc/SITE.rc)
BCSTAG="Icarus_Reynolds"

case $SITE in
   NCCS)
   SCMDIR="/discover/swdev/gmao_SIteam/scm/scminfiles/git-v11.1/"
   BCSDIR="/discover/nobackup/projects/gmao/share/gmao_ops/fvInput/g5gcm/bcs/Icarus/$BCSTAG"
   CHMDIR="/discover/nobackup/projects/gmao/share/gmao_ops/fvInput_nc3"
   ;;
   NAS)
   SCMDIR="/nobackup/gmao_SIteam/ModelData/scminfiles/git-v11.1/"
   BCSDIR="/nobackup/gmao_SIteam/ModelData/bcs/Icarus/$BCSTAG"
   CHMDIR="/nobackup/gmao_SIteam/ModelData/fvInput_nc3"
   ;;
   GMAO*)
   SCMDIR="/ford1/share/gmao_SIteam/ModelData/scminfiles/git-v11.1/"
   BCSDIR="/ford1/share/gmao_SIteam/ModelData/bcs/Icarus/$BCSTAG"
   CHMDIR="/ford1/share/gmao_SIteam/ModelData/fvInput_nc3"
   ;;
   *)
   echo "Unknown site $SITE detected."

   # 1. Prompt the user for the TinyBCs installation path
   if [[ ! -d "$TINYBCS_PATH" ]]; then
      read -p "Please enter the full path to your TinyBCs installation: " TINYBCS_PATH
   fi

   # 2. Verify the path exists and is a directory
   if [[ ! -d "$TINYBCS_PATH" ]]; then
      echo "ERROR: The directory '$TINYBCS_PATH' does not exist."
      exit 3
   fi

   # 3. Set the required directory variables based on the user input
   SCMDIR="$TINYBCS_PATH/scm"
   BCSDIR="$TINYBCS_PATH/bcs/Icarus"
   CHMDIR="$TINYBCS_PATH/chem"

   # 4. Check if the selected case exists in this new SCMDIR
   if [[ ! -d "$SCMDIR/$selected_case" ]]; then
      echo "ERROR: Case '$selected_case' not found in $SCMDIR"

      if [[ -d "$SCMDIR" ]]; then
         echo "The available cases in this installation are:"
         ls -1 "$SCMDIR"
      else
         echo "Additionally, the expected SCM directory ($SCMDIR) was not found."
      fi

      exit 1
   fi

   echo "Proceeding with TinyBCs paths at $TINYBCS_PATH"
   # We should also tell the user that SCM in TinyBCs has only
   # been tested for 7 days of simulation. So they should alter
   # CAP.rc JOB_SGMT to 'JOB_SGMT:     00000007 000000'
   echo "Note: SCM in TinyBCs has only been tested for 7 days of simulation."
   echo "      Please ensure that CAP.rc JOB_SGMT is set to 'JOB_SGMT: 00000007 000000'"
   ;;
esac

SOURCEDIR=$SCMDIR/$selected_case

mkdir -p $expdir

/bin/cp $INSTALLDIR/bin/GEOSgcm.x $expdir
/bin/cp $INSTALLDIR/etc/logging.yaml $expdir

/bin/cp $SOURCEDIR/$DATFILE $expdir
/bin/cp $SOURCEDIR/topo_dynave.data $expdir
/bin/cp $SOURCEDIR/fraci.data $expdir
/bin/cp $SOURCEDIR/SEAWIFS_KPAR_mon_clim.data $expdir
/bin/cp $SOURCEDIR/sst.data $expdir
/bin/cp $SOURCEDIR/sstsi.data $expdir
/bin/cp $SOURCEDIR/tile.data $expdir
/bin/cp $SOURCEDIR/topo_gwdvar.data $expdir
/bin/cp $SOURCEDIR/topo_trbvar.data $expdir
/bin/cp $SOURCEDIR/lai.data $expdir
/bin/cp $SOURCEDIR/green.data $expdir
/bin/cp $SOURCEDIR/ndvi.data $expdir
/bin/cp $SOURCEDIR/nirdf.dat $expdir
/bin/cp $SOURCEDIR/vegdyn.data $expdir
/bin/cp $SOURCEDIR/visdf.dat $expdir
#/bin/cp $SOURCEDIR/*rst $expdir

/bin/cp $SOURCEDIR/datmodyn_internal_rst $expdir
/bin/cp $SOURCEDIR/moist_internal_rst $expdir
/bin/cp $SOURCEDIR/catch_internal_rst $expdir
/bin/cp $SOURCEDIR/gwd_internal_rst $expdir

mkdir -p $expdir/ExtData

/bin/cp $SOURCEDIR/CAP.rc $expdir
/bin/cp $SOURCEDIR/cap_restart $expdir
/bin/cp $SOURCEDIR/HISTORY.rc $expdir

#/bin/ln -sf $BCSDIR/Shared/pchem.species.CMIP-5.1870-2097.z_91x72.nc4 $expdir/species.data
/bin/ln -sf $BCSDIR/Shared/pchem.species.CMIP-5.MERRA2OX.197902-201706.z_91x72.nc4 $expdir/species.data

/bin/cp $INSTALLDIR/etc/*.rc $expdir

# Need to handle ExtData. At the moment, it is not known how to run
# GOCART2G.data, so we default to GOCART2G AMIP

year=$(cat $expdir/cap_restart | cut -c1-4)

/bin/cp $INSTALLDIR/etc/*.yaml $expdir
/bin/cp $INSTALLDIR/etc/AMIP/*.yaml $expdir
/bin/mv $expdir/WSUB_ExtData.yaml $expdir/WSUB_ExtData.tmp
cat $expdir/WSUB_ExtData.tmp | sed '/collection:/ s#WSUB_SWclim.*#/dev/null#' > $expdir/WSUB_ExtData.yaml
/bin/rm $expdir/WSUB_ExtData.tmp

/bin/cp $SCMDIR/general/* $expdir

/bin/cp $INSTALLDIR/etc/AGCM.rc.tmpl $expdir

/bin/cp $SOURCEDIR/sedfile.CASE $expdir
/bin/cp $SOURCEDIR/AGCM.CASE.txt $expdir

# These are the weights files
/bin/cp $SOURCEDIR/rh_* $expdir

# MAT These sed commands have to be done in the experiment directory.
#     This should probably be fixed at some point

builtin cd $expdir
$SED -f sedfile.SCM AGCM.rc.tmpl > AGCM.rc.sed.general
$SED -f sedfile.CASE AGCM.rc.sed.general > AGCM.rc.CASE
awk '{ if ( $1  !~ "#DELETE") { print } }' AGCM.rc.CASE > AGCM.rc
/bin/rm AGCM.rc.sed.general AGCM.rc.CASE
builtin cd $CURRDIR

if [ $nlev -ne 72 ]
then

  digits=0000
  temp=$digits$nlev
  padlev=${temp:(-${#digits})}

  $ISED -e "s/L72/L${nlev}/g" -e "s/z72/z${nlev}/g" $expdir/*.rc
  $ISED -e "s/L72/L${nlev}/g" -e "s/z72/z${nlev}/g" $expdir/*.yaml

  $ISED "/AGCM_LM:/c\AGCM_LM: ${nlev}" $expdir/AGCM.rc
  $ISED "/AGCM.LM:/c\AGCM.LM: ${nlev}" $expdir/AGCM.rc
  $ISED "/SCMGRID.LM:/c\SCMGRID.LM: ${nlev}" $expdir/HISTORY.rc
  cp $SOURCEDIR/*rst.r0001x0001x${padlev} $expdir
  /bin/mv $expdir/datmodyn_internal_rst.r0001x0001x${padlev} $expdir/datmodyn_internal_rst
  /bin/mv $expdir/moist_internal_rst.r0001x0001x${padlev} $expdir/moist_internal_rst

fi

/bin/ln -s $CHMDIR/g5chem $expdir/ExtData
/bin/ln -s $CHMDIR/PIESA $expdir/ExtData
/bin/ln -s $CHMDIR/g5gcm $expdir/ExtData
/bin/ln -s $CHMDIR/chemistry $expdir/ExtData

# Handle scm_run.j

/bin/cp $INSTALLDIR/bin/scm_run.j $expdir

if [[ "$SITE" == NAS ]]
then
   BATCH_TIME="PBS -l walltime=00:30:00"
   BATCH_GROUP="PBS -W group_list=$account"
   BATCH_JOBNAME="PBS -N $selected_case"
   BATCH_TASKS="PBS -l select=1:ncpus=1:mpiprocs=1:model=cas_ait"
else
   BATCH_TIME="SBATCH --time=00:30:00"
   BATCH_GROUP="SBATCH --account=$account"
   BATCH_JOBNAME="SBATCH --job-name=$selected_case"
   BATCH_TASKS="SBATCH --ntasks=1"
fi

cat > $expdir/sedfile << EOF
s?@EXPDIR?$expdir?g
s?@SITE?$SITE?g
s?@INSTALLDIR?$INSTALLDIR?g
s?@BATCH_TASKS?$BATCH_TASKS?g
s?@BATCH_TIME?$BATCH_TIME?g
s?@BATCH_GROUP?$BATCH_GROUP?g
s?@BATCH_JOBNAME?$BATCH_JOBNAME?g
s?@PRELOAD_COMMAND?$PRELOAD_COMMAND?g
s?@LD_LIBRARY_PATH_CMD?$LD_LIBRARY_PATH_CMD?g
s?@RUN_CMD?$RUN_CMD?g
s?@FILE_WEIGHTS?$FILE_WEIGHTS?g
EOF

$ISED -f $expdir/sedfile $expdir/scm_run.j
#/bin/rm $expdir/sedfile

echo "SUCCESS! The $selected_case experiment has been created in $expdir"
echo
exit 0