From e06284eecae44ed731bf7975c5c963e8f1d021d4 Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Mon, 29 Apr 2024 12:11:23 -0400 Subject: [PATCH 1/6] bring python3 to FP --- CHANGELOG.md | 5 +++++ g5_modules | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45cf642..7abae8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed ### Added +## [3.4.0.1] - 2024-Apr-29 + +### Changed + - Min4.11.0_py3.9_AND_Min4.8.3_py2.7 + ## [3.4.0] - 2021-Oct-01 ### Changed diff --git a/g5_modules b/g5_modules index 73d8e67..95f8052 100755 --- a/g5_modules +++ b/g5_modules @@ -148,7 +148,7 @@ if ( $site == NCCS ) then set mod4 = mpi/impi/2021.2.0 - set mod5 = python/GEOSpyD/Min4.8.3_py2.7 + set mod5 = python/GEOSpyD/Min4.11.0_py3.9_AND_Min4.8.3_py2.7 set basedir = /discover/swdev/gmao_SIteam/Baselibs/ESMA-Baselibs-6.2.7/x86_64-pc-linux-gnu/ifort_2021.2.0-intelmpi_2021.2.0 @@ -182,7 +182,7 @@ else if ( $site == NAS ) then endif set mod3 = comp-intel/2021.2.0 set mod4 = mpi-hpe/mpt.2.23 - set mod5 = python/GEOSpyD/Min4.8.3_py2.7 + set mod5 = python/GEOSpyD/Min4.11.0_py3.9_AND_Min4.8.3_py2.7 set mods = ( $mod1 $mod2 $mod3 $mod4 $mod5 ) set modinit = /usr/share/modules/init/tcsh @@ -259,7 +259,7 @@ else if ( $site == GMAO.desktop ) then set mod2 = comp/gcc/10.1.0 set mod3 = comp/intel/2021.2.0 set mod4 = mpi/impi/2021.2.0 - set mod5 = other/python/GEOSpyD/Min4.8.3_py2.7 + set mod5 = other/python/GEOSpyD/Min4.11.0_py3.9_AND_Min4.8.3_py2.7 set mods = ( $mod1 $mod2 $mod3 $mod4 $mod5 ) set modinit = /usr/share/Modules/init/tcsh From 298eb2ed06986b3a3e65e234c7fed9028c2f5e9d Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Fri, 24 Jan 2025 14:29:40 -0500 Subject: [PATCH 2/6] Updates for GEOSadas 5.29.5 on SLES15 --- .editorconfig | 32 ++++ CHANGELOG.md | 9 +- build.csh | 413 +++++++++++++++++++++++++++++++++++--------------- g5_modules | 34 +++-- 4 files changed, 351 insertions(+), 137 deletions(-) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..9f1cd65 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,32 @@ +# Global Editor Config for MAPL +# +# This is an ini style configuration. See http://editorconfig.org/ for more information on this file. +# +# Top level editor config. +root = true + +# Always use Unix style new lines with new line ending on every file and trim whitespace +[*] +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +# Python: PEP8 defines 4 spaces for indentation +[*.py] +indent_style = space +indent_size = 4 + +# YAML format, 2 spaces +[{*.yaml,*.yml}] +indent_style = space +indent_size = 2 + +# CMake (from KitWare: https://github.com/Kitware/CMake/blob/master/.editorconfig) +[{CMakeLists.txt,*.cmake,*.rst}] +indent_style = space +indent_size = 2 + +# Markdown +[*.md] +trim_trailing_whitespace = true +indent_style = space diff --git a/CHANGELOG.md b/CHANGELOG.md index 7abae8d..a7124ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,10 +12,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed ### Added +## [3.4.0.2] - 2025-01-24 + +### Changed + +- Updates necessary to build on SLES15 at NCCS + ## [3.4.0.1] - 2024-Apr-29 ### Changed - - Min4.11.0_py3.9_AND_Min4.8.3_py2.7 + + - Update to Min4.11.0_py3.9_AND_Min4.8.3_py2.7 on SLES12 ## [3.4.0] - 2021-Oct-01 diff --git a/build.csh b/build.csh index 1e916d7..fc997b4 100755 --- a/build.csh +++ b/build.csh @@ -20,11 +20,11 @@ # 05Aug2009 Stassi Added -debug and -tmpdir flags # 19Aug2010 Stassi Added -walltime flag # 01Apr2011 Stassi Added clean and realclean options -# 04Nov2014 MAT Moved to sbatch on discover, added Haswell +# 04Nov2014 MAT Moved to sbatch on discover, added Haswell # as an option, removed some older batch systems # 07Jul2016 MAT Added Broadwell at NAS. Removed Westmere. Made # Broadwell default at NAS. -# 10Oct2017 MAT Added Skylake at NAS. Added option to pass in +# 10Oct2017 MAT Added Skylake at NAS. Added option to pass in # account # 08Jul2019 MAT Changes for git-based GEOSadas #------------------------------------------------------------------------ @@ -42,7 +42,7 @@ set time = ( 1000 "%Uu %Ss %E" ) set NCPUs_min = 6 #===================== -# determine the site +# determine the site #===================== set node = `uname -n` if ( ($node == dirac) \ @@ -51,9 +51,10 @@ if ( ($node == dirac) \ || ($node =~ discover*)) then setenv SITE NCCS -else if (($node =~ pfe*) \ - || ($node =~ bridge*) \ - || ($node =~ r[0-9]*i[0-9]*n[0-9]*)) then +else if (($node =~ pfe*) \ + || ($node =~ afe*) \ + || ($node =~ r[0-9]*i[0-9]*n[0-9]*) \ + || ($node =~ r[0-9]*c[0-9]*t[0-9]*n[0-9]*)) then setenv SITE NAS else @@ -61,32 +62,33 @@ else set NCPUs_min = 1 endif -# if batch, then skip over job submission -#---------------------------------------- -if ($?Parallel_build_bypass_flag) then - goto build -endif - # set defaults #------------- -setenv esmadir "" -setenv docmake 1 -setenv usegnu 0 -setenv usehydro 0 -setenv usenonhydro 0 -setenv ddb 0 -setenv debug 0 -setenv aggressive 0 -setenv verbose "" -setenv interactive 0 -setenv proc "" -setenv prompt 1 -setenv queue "" -setenv partition "" -setenv account "" -setenv tmpdir "" -setenv walltime "" -setenv cmake_build_type "Release" +if (! $?ESMADIR) setenv ESMADIR "" +if (! $?docmake) setenv docmake 1 +if (! $?usegnu) setenv usegnu 0 +if (! $?notar) setenv notar 0 +if (! $?ddb) setenv ddb 0 +if (! $?debug) setenv debug 0 +if (! $?aggressive) setenv aggressive 0 +if (! $?verbose) setenv verbose "" +if (! $?interactive) setenv interactive 0 +if (! $?do_wait) setenv do_wait 0 +if (! $?proc) setenv proc "" +if (! $?prompt) setenv prompt 1 +if (! $?queue) setenv queue "" +if (! $?partition) setenv partition "" +if (! $?account) setenv account "" +if (! $?tmpdir) setenv tmpdir "" +if (! $?walltime) setenv walltime "" +if (! $?slurm_constraint) setenv slurm_constraint "" +if (! $?cmake_build_type) setenv cmake_build_type "Release" +if (! $?EXTRA_CMAKE_FLAGS) setenv EXTRA_CMAKE_FLAGS "" +if (! $?FORTRAN_COMPILER) setenv FORTRAN_COMPILER "" +if (! $?BUILDDIR_PASSED) setenv BUILDDIR_PASSED "NO" +if (! $?INSTALLDIR_PASSED) setenv INSTALLDIR_PASSED "NO" +if (! $?usehydro) setenv usehydro 0 +if (! $?usenonhydro) setenv usenonhydro 0 # Detect if on compute node already # --------------------------------- @@ -113,23 +115,25 @@ while ($#argv) #------------------- if (("$1" == "-debug") || ("$1" == "-db")) then setenv cmake_build_type "Debug" - set debug = 1 + setenv debug 1 endif # compile with aggressive #------------------------ if ("$1" == "-aggressive") then setenv cmake_build_type "Aggressive" - set aggressive = 1 + setenv aggressive 1 endif # specify node type #------------------ + if ("$1" == "-mil") set nodeTYPE = "Milan" if ("$1" == "-rom") set nodeTYPE = "Rome" if ("$1" == "-cas") set nodeTYPE = "CascadeLake" if ("$1" == "-sky") set nodeTYPE = "Skylake" if ("$1" == "-bro") set nodeTYPE = "Broadwell" if ("$1" == "-has") set nodeTYPE = "Haswell" + if ("$1" == "-any") set nodeTYPE = "Any node" # reset Fortran TMPDIR #--------------------- @@ -150,6 +154,7 @@ while ($#argv) if ("$1" == "-builddir") then shift; if (! $#argv) goto usage setenv BUILDDIR $1 + setenv BUILDDIR_PASSED "YES" endif # set INSTALLDIR @@ -157,17 +162,29 @@ while ($#argv) if ("$1" == "-installdir") then shift; if (! $#argv) goto usage setenv INSTALLDIR $1 + setenv INSTALLDIR_PASSED "YES" + endif + + # set GMI_MECHANISM + #------------------ + if ("$1" == "-gmi_mechanism") then + shift; if (! $#argv) goto usage + setenv GMI_MECHANISM $1 endif # run job interactively #---------------------- if ("$1" == "-i") set interactive = 1 - # run job interactively - #---------------------- + # set verbose flag + #----------------- if ("$1" == "-verbose") set verbose = "VERBOSE=1" if ("$1" == "-v") set verbose = "VERBOSE=1" + # run job interactively + #---------------------- + if ("$1" == "-wait") set do_wait = 1 + # submit batch job to alternative queue/qos #------------------------------------------ if ("$1" == "-q") then @@ -226,6 +243,12 @@ while ($#argv) setenv usegnu 1 endif + # set no tar option + #------------------ + if ("$1" == "-no-tar") then + setenv notar 1 + endif + # set hydrostatic option #----------------------- if ("$1" == "-hydrostatic") then @@ -238,9 +261,27 @@ while ($#argv) setenv usenonhydro 1 endif + # If a user passes in '-- ' then everything after that is passed + # into EXTRA_CMAKE_FLAGS and we are done parsing arguments + #-------------------------------------------------------------- + if ("$1" == "--") then + shift + setenv EXTRA_CMAKE_FLAGS "$*" + set argv = () + break + endif + shift end +# Check if the ESMA_NOTAR environment variable is set +# If so, then set notar to 1 +# -------------------------------------------------- +if ($?ESMA_NOTAR) then + echo "ESMA_NOTAR is set, so not creating tar file" + setenv notar 1 +endif + # Only allow one of debug and aggressive # -------------------------------------- if ( ($aggressive) && ($debug) ) then @@ -258,7 +299,7 @@ endif # default nodeTYPE #----------------- if (! $?nodeTYPE) then - if ($SITE == NCCS) set nodeTYPE = "Skylake" + if ($SITE == NCCS) set nodeTYPE = "Any" if ($SITE == NAS) set nodeTYPE = "Skylake" endif @@ -267,18 +308,39 @@ endif if ($SITE == NCCS) then set nT = `echo $nodeTYPE| tr "[A-Z]" "[a-z]" | cut -c1-3 ` - if (($nT != has) && ($nT != sky) && ($nT != cas)) then + if (($nT != sky) && ($nT != cas) && ($nT != mil) && ($nT != any)) then echo "ERROR. Unknown node type at NCCS: $nodeTYPE" exit 1 endif - if ($nT == has) @ NCPUS_DFLT = 28 + # For the any node, set the default to 40 cores as + # this is the least number of cores you will get + if ($nT == any) @ NCPUS_DFLT = 40 if ($nT == sky) @ NCPUS_DFLT = 40 if ($nT == cas) @ NCPUS_DFLT = 48 + if ($nT == mil) @ NCPUS_DFLT = 128 - if ($nT == has) set proc = 'hasw' + if ($nT == any) set proc = 'any' if ($nT == sky) set proc = 'sky' if ($nT == cas) set proc = 'cas' + if ($nT == mil) set proc = 'mil' + + # If we are using GNU at NCCS, we can*only* use the cas or mil processors + # as OpenMPI is only built for Infiniband + if ($usegnu) then + if ($nT == mil) then + echo "Using GNU at NCCS, setting queue to cas" + set proc = 'mil' + else + echo "Using GNU at NCCS, setting queue to cas" + set proc = 'cas' + endif + set slurm_constraint = "--constraint=$proc" + else if ($nT == any) then + set slurm_constraint = "--constraint=sky|cas" + else + set slurm_constraint = "--constraint=$proc" + endif if ("$queue" == "") then set queue = '--qos=debug' @@ -300,7 +362,7 @@ if ( $SITE == NAS ) then exit 2 endif - if ($nT == rom) set nT = 'rom_ait:aoe=sles15' + if ($nT == rom) set nT = 'rom_ait' if ($nT == sky) set nT = 'sky_ele' if ($nT == cas) set nT = 'cas_ait' set proc = ":model=$nT" @@ -309,11 +371,11 @@ if ( $SITE == NAS ) then if ($nT == bro) @ NCPUS_DFLT = 28 if ($nT == sky_ele) @ NCPUS_DFLT = 40 if ($nT == cas_ait) @ NCPUS_DFLT = 40 - if ($nT == "rom_ait:aoe=sles15") @ NCPUS_DFLT = 128 + if ($nT == rom_ait) @ NCPUS_DFLT = 128 # TMPDIR needs to be reset #------------------------- - if (! "$tmpdir") then + if ($tmpdir == '') then set tmpdirDFLT = "/nobackup/$USER/scratch/" if ($prompt) then echo "" @@ -322,6 +384,8 @@ if ( $SITE == NAS ) then echo -n "TMPDIR [$tmpdirDFLT] " setenv tmpdir $< if ("$tmpdir" == "") setenv tmpdir $tmpdirDFLT + else + setenv tmpdir $tmpdirDFLT endif endif echo "TMPDIR: $tmpdir" @@ -341,26 +405,90 @@ endif # set Pbuild_build_directory # -------------------------- -if ($?BUILDDIR) then - setenv Pbuild_build_directory $ESMADIR/$BUILDDIR -else if ($debug) then - setenv Pbuild_build_directory $ESMADIR/build-Debug -else if ($aggressive) then - setenv Pbuild_build_directory $ESMADIR/build-Aggressive -else - setenv Pbuild_build_directory $ESMADIR/build +if (! $?Pbuild_build_directory) then + if ($?BUILDDIR) then + setenv Pbuild_build_directory $ESMADIR/$BUILDDIR + else if ($debug) then + setenv Pbuild_build_directory $ESMADIR/build-Debug + else if ($aggressive) then + setenv Pbuild_build_directory $ESMADIR/build-Aggressive + else + setenv Pbuild_build_directory $ESMADIR/build + endif endif # set Pbuild_install_directory # ---------------------------- -if ($?INSTALLDIR) then - setenv Pbuild_install_directory $ESMADIR/$INSTALLDIR -else if ($debug) then - setenv Pbuild_install_directory $ESMADIR/install-Debug -else if ($aggressive) then - setenv Pbuild_install_directory $ESMADIR/install-Aggressive -else - setenv Pbuild_install_directory $ESMADIR/install +if (! $?Pbuild_install_directory) then + if ($?INSTALLDIR) then + setenv Pbuild_install_directory $ESMADIR/$INSTALLDIR + else if ($debug) then + setenv Pbuild_install_directory $ESMADIR/install-Debug + else if ($aggressive) then + setenv Pbuild_install_directory $ESMADIR/install-Aggressive + else + setenv Pbuild_install_directory $ESMADIR/install + endif +endif + +# If we are at NCCS, because of the dual OSs, we decorate the build and +# install directory with the OS name. If we submit to Milan, we will add +# -SLES15, otherwise -SLES12 to the build and install directories. But, +# we only do this if the user has not specified a build directory or +# install directory +# --------------------------------------------------------------------- + +if ($SITE == NCCS) then + # We now have to handle this in two ways. One if we are on a compute node and one if we aren't. + # This is because of how this script works where it sort of submits itself to the batch system + # and many of the variables known by the script before submission are lost after submission. + # So if we are on a compute node, we detect the OS version directly, but if we are just submitting on a + # head node, we instead have to just use the processor type passed in. We'll use oncompnode to detect + # which case we are in. + if ($oncompnode) then + set OS_VERSION=`grep VERSION_ID /etc/os-release | cut -d= -f2 | cut -d. -f1 | sed 's/"//g'` + else + if ($nT == mil) then + set OS_VERSION = 15 + else + set OS_VERSION = 12 + endif + endif + # We also check if we already appended SLES + if (! $?BUILDDIR && "$BUILDDIR_PASSED" == "NO") then + if ($Pbuild_build_directory !~ "*-SLES${OS_VERSION}") then + setenv Pbuild_build_directory ${Pbuild_build_directory}-SLES${OS_VERSION} + endif + endif + if (! $?INSTALLDIR && "$INSTALLDIR_PASSED" == "NO") then + if ($Pbuild_install_directory !~ "*-SLES${OS_VERSION}") then + setenv Pbuild_install_directory ${Pbuild_install_directory}-SLES${OS_VERSION} + endif + endif +endif + +if ("$FORTRAN_COMPILER" == "") then + if ($usegnu) then + setenv FORTRAN_COMPILER 'gfortran' + else + setenv FORTRAN_COMPILER 'ifort' + endif +endif + +if (! $?INSTALL_SOURCE_TARFILE) then + if ($notar) then + setenv INSTALL_SOURCE_TARFILE "OFF" + else + setenv INSTALL_SOURCE_TARFILE "ON" + endif +endif + +if (! $?GMI_MECHANISM_FLAG) then + if ($?GMI_MECHANISM) then + setenv GMI_MECHANISM_FLAG "-DGMI_MECHANISM=$GMI_MECHANISM" + else + setenv GMI_MECHANISM_FLAG "" + endif endif # developer's debug @@ -373,21 +501,34 @@ if ($ddb) then if ($?nodeTYPE) then echo "nodeTYPE = $nodeTYPE" endif + if ($?GMI_MECHANISM) then + echo "GMI_MECHANISM = $GMI_MECHANISM" + endif echo "tmpdir = $tmpdir" echo "proc = $proc" echo "interactive = $interactive" + echo "do_wait = $do_wait" echo "queue = $queue" if ($SITE == NCCS) then echo "partition = $partition" + echo "slurm_constraint = $slurm_constraint" endif echo "account = $account" echo "walltime = $walltime" echo "prompt = $prompt" echo "nocmake = $docmake" + echo "notar = $notar" echo "NCPUS_DFLT = $NCPUS_DFLT" echo "CMAKE_BUILD_TYPE = $cmake_build_type" + echo "EXTRA_CMAKE_FLAGS = $EXTRA_CMAKE_FLAGS" echo "Build directory = $Pbuild_build_directory" echo "Install directory = $Pbuild_install_directory" + echo "usegnu = $usegnu" + echo "FORTRAN_COMPILER = $FORTRAN_COMPILER" + echo "INSTALL_SOURCE_TARFILE = $INSTALL_SOURCE_TARFILE" + echo "GMI_MECHANISM_FLAG = $GMI_MECHANISM_FLAG" + echo "BUILDDIR_PASSED = $BUILDDIR_PASSED" + echo "INSTALLDIR_PASSED = $INSTALLDIR_PASSED" exit endif @@ -409,7 +550,7 @@ if ("$tmpdir" != "") then # ... check that it is writeable #------------------------------- if (! -w $tmpdir) then - echo ">> Error << TMPDIR is not writeable: $tmpdir" + echo ">> Error << TMPDIR is not writeable: $tmpdir" exit 4 endif echo "" @@ -429,15 +570,6 @@ echo " PARALLEL BUILD " echo " ================" echo "" -# set environment variables -#-------------------------- -if ( -d ${ESMADIR}/@env ) then - source $ESMADIR/@env/g5_modules -else if ( -d ${ESMADIR}/env@ ) then - source $ESMADIR/env@/g5_modules -else if ( -d ${ESMADIR}/env ) then - source $ESMADIR/env/g5_modules -endif setenv Pbuild_source_directory $ESMADIR # Make the BUILD directory @@ -451,7 +583,6 @@ if (! -d $Pbuild_build_directory) then endif endif -setenv Parallel_build_bypass_flag set jobname = "parallel_build" #=========================== @@ -534,36 +665,37 @@ setenv bldlogdir $Pbuild_build_directory/$BUILD_LOG_DIR setenv cmakelog $bldlogdir/CLOG setenv buildlog $bldlogdir/LOG setenv buildinfo $bldlogdir/info -setenv cleanFLAG "" +if (! $?cleanFLAG ) setenv cleanFLAG "notset" -ls $cmakelog $buildlog $buildinfo >& /dev/null -if ($status == 0) then - if ($prompt) then - echo '' - echo 'Previous build detected - Do you want to clean?' - echo '(c) clean: runs "make clean"' - echo '(r) realclean: removes build directory and re-runs CMake' - echo '(n) no clean' - echo '' - echo -n 'Select (c,r,n) <> ' - - set do_clean = $< - if ("$do_clean" != "r" && "$do_clean" != "n") then +if ("$cleanFLAG" == "notset") then + ls $cmakelog $buildlog $buildinfo >& /dev/null + if ($status == 0) then + if ($prompt) then + echo '' + echo 'Previous build detected - Do you want to clean?' + echo '(c) clean: Removes build and install directories, and rebuilds' + echo '(n) no clean' + echo '' + echo " Note: if you have changed MAPL, we recommend doing a clean for safety's sake" + echo '' + echo -n 'Select (c,n) <> ' + + set do_clean = $< + if ("$do_clean" != "n") then + set do_clean = "c" + endif + else set do_clean = "c" endif - else - set do_clean = "c" - endif - if ("$do_clean" == "c") then - setenv cleanFLAG clean - echo "make clean before rebuild" - else if ("$do_clean" == "r") then - setenv cleanFLAG realclean - echo "remove build directory and re-run CMake before rebuild" - else - echo "No clean before rebuild" - endif + if ("$do_clean" == "c") then + setenv cleanFLAG "clean" + echo "Removing build and install directories and re-running CMake before rebuild" + else + setenv cleanFLAG "noclean" + echo "No clean before rebuild" + endif + endif endif #============== @@ -626,10 +758,19 @@ else if (-e `which getsponsor` && (! $interactive)) then set groupflag = "--account=$group" endif +set waitflag = "" +if ($do_wait) then + if ($SITE == NAS) then + set waitflag = "-W block=true" + else if ($SITE == NCCS) then + set waitflag = "--wait" + endif +endif + if ($interactive) then goto build else if ( $SITE == NAS ) then - if ("$walltime" == "") setenv walltime "1:00:00" + if ("$walltime" == "") setenv walltime "1:30:00" set echo qsub $groupflag $queue \ -N $jobname \ @@ -637,25 +778,36 @@ else if ( $SITE == NAS ) then -l walltime=$walltime \ -S /bin/csh \ -V -j oe -k oed \ + $waitflag \ $0 unset echo - sleep 1 - qstat -a | grep $USER + if ("$waitflag" == "") then + sleep 1 + qstat -a | grep $USER + endif else if ( $SITE == NCCS ) then if ("$walltime" == "") setenv walltime "1:00:00" set echo - sbatch $groupflag $partition $queue \ - --constraint=$proc \ + # NOTE: The weird long export line below is needed at NCCS because of the + # two OSs. For some reason, if you submit a Milan job from a SLES12 + # headnode, it was seeing SLES12 module paths. We believe this is + # because SLURM by default exports all the environment + sbatch $groupflag $partition $queue \ + $slurm_constraint \ --job-name=$jobname \ --output=$jobname.o%j \ --nodes=1 \ --ntasks=${numjobs} \ --time=$walltime \ + --export ESMADIR=${ESMADIR},cmake_build_type=${cmake_build_type},EXTRA_CMAKE_FLAGS=${EXTRA_CMAKE_FLAGS},FORTRAN_COMPILER=${FORTRAN_COMPILER},INSTALL_SOURCE_TARFILE=${INSTALL_SOURCE_TARFILE},verbose=${verbose},GMI_MECHANISM_FLAG=${GMI_MECHANISM_FLAG},Pbuild_build_directory=${Pbuild_build_directory},Pbuild_install_directory=${Pbuild_install_directory},usegnu=${usegnu},notar=${notar},tmpdir=${tmpdir},docmake=${docmake},debug=${debug},aggressive=${aggressive},BUILDDIR_PASSED=${BUILDDIR_PASSED},INSTALLDIR_PASSED=${INSTALLDIR_PASSED},queue=${queue},partition=${partition},cleanFLAG=${cleanFLAG} \ + $waitflag \ $0 unset echo - sleep 1 - # Add a longer format for the job name for scripting purposes - squeue -a -o "%.10i %.12P %.10q %.30j %.8u %.8T %.10M %.9l %.6D %.6C %R" -u $USER + if ("$waitflag" == "") then + sleep 1 + # Add a longer format for the job name for scripting purposes + squeue -a -o "%.10i %.12P %.10q %.30j %.8u %.8T %.10M %.9l %.6D %.6C %R" -u $USER + endif else echo $scriptname": batch procedures are not yet defined for node=$node at site=$SITE" endif @@ -668,11 +820,14 @@ build: # BUILD SYSTEM #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -if ( $cleanFLAG == "realclean" ) then +if ( $cleanFLAG == "clean" ) then + echo "Removing build and install directories" rm -rf $Pbuild_build_directory rm -rf $Pbuild_install_directory mkdir -p $Pbuild_build_directory +else + echo "Using existing build and install directories" endif chdir $Pbuild_build_directory @@ -716,8 +871,9 @@ echo "Writing LOG and info files to directory: $bldlogdir:t" echo2 "" #================ -# set environment +# set environment #================ + if ( -d ${ESMADIR}/@env ) then source $ESMADIR/@env/g5_modules else if ( -d ${ESMADIR}/env@ ) then @@ -744,8 +900,13 @@ endif if ("$queue" != "") then echo1 "queue: $queue" endif +if ("$partition" != "") then + echo1 "partition: $partition" +endif if ("$account" != "") then echo1 "account: $account" +echo1 "Pbuild_build_directory: $Pbuild_build_directory" +echo1 "Pbuild_install_directory: $Pbuild_install_directory" endif echo1 "======================================" @@ -755,24 +916,6 @@ echo1 "======================================" #=============== # build system #=============== -if ( $cleanFLAG == "clean" ) then - echo1 "" - echo1 "--------------------------------------" - date1 - echo1 "make $cleanFLAG" - make $cleanFLAG - set buildstatus = $status - echo1 "clean complete; status = $buildstatus" - date1 - echo1 "--------------------------------------" - time >> $buildinfo -endif - -if ($usegnu) then - setenv FORTRAN_COMPILER 'gfortran' -else - setenv FORTRAN_COMPILER 'ifort' -endif if ($usehydro) then setenv HYDROBUILD '-DHYDROSTATIC=ON' @@ -783,9 +926,9 @@ else setenv HYDROBUILD '' endif -set cmd1 = "cmake $ESMADIR -DCMAKE_INSTALL_PREFIX=$Pbuild_install_directory -DBASEDIR=${BASEDIR}/${ARCH} -DCMAKE_Fortran_COMPILER=${FORTRAN_COMPILER} -DCMAKE_BUILD_TYPE=${cmake_build_type} ${HYDROBUILD}" +set cmd1 = "cmake $ESMADIR -DCMAKE_INSTALL_PREFIX=$Pbuild_install_directory -DBASEDIR=${BASEDIR}/${ARCH} -DCMAKE_Fortran_COMPILER=${FORTRAN_COMPILER} -DCMAKE_BUILD_TYPE=${cmake_build_type} ${HYDROBUILD} -DINSTALL_SOURCE_TARFILE=${INSTALL_SOURCE_TARFILE} ${GMI_MECHANISM_FLAG} ${EXTRA_CMAKE_FLAGS}" set cmd2 = "make --jobs=$numjobs install $verbose" -echo1 "" +echo1 "" echo1 "" if ($docmake) then echo1 "--------------------------------------" @@ -836,6 +979,8 @@ flagged options -esmadir dir esmadir location -nocmake do not run cmake (useful for scripting) -gnu build with gfortran + -wait wait when run as a batch job + -no-tar build with INSTALL_SOURCE_TARFILE=OFF (does not tar up source tarball, default is ON) -hydrostatic build for hydrostatic dynamics in FV -nonhydrostatic build for nonhydrostatic dynamics in FV @@ -846,9 +991,25 @@ flagged options -account account send batch job to account -walltime hh:mm:ss time to use as batch walltime at job submittal + -mil compile on Milan nodes (only at NCCS) -rom compile on Rome nodes (only at NAS) -cas compile on Cascade Lake nodes - -sky compile on Skylake nodes (default) + -sky compile on Skylake nodes (default at NAS) -bro compile on Broadwell nodes (only at NAS) - -has compile on Haswell nodes + -has compile on Haswell nodes (only at NAS) + -any compile on either Sky or Cascade Lake node (only at NCCS with SLURM, default at NCCS) + +extra cmake options + + To pass in additional CMake options not covered by the above flags, + after all your flags, add -- and then the options. For example: + + $scriptname -debug -- -DSTRATCHEM_REDUCED_MECHANISM=ON -DUSE_CODATA_2018_CONSTANTS=ON + + and these options will be appended to the CMake command. + + NOTE: Once you use --, you cannot use any more flags. All options + after -- will be passed to CMake and if not a valid CMake option, + could cause the build to fail. + EOF diff --git a/g5_modules b/g5_modules index 95f8052..78f98da 100755 --- a/g5_modules +++ b/g5_modules @@ -41,7 +41,7 @@ # 21Jul2008 Takacs New modules and BASEDIR on discover after OS upgrade # 13Apr2009 Stassi Updated for pleiades # 22Apr2010 Kokron Updated for Fortuna-2.1 on pleiades -# 21Jul2011 Kokron Overlay older MKL module as on discover to gain reproducible results from dgeev in GSI +# 21Jul2011 Kokron Overlay older MKL module as on discover to gain reproducible results from dgeev in GSI # 24Aug2012 Stassi Added sh option to write bash source-able file # 03Nov2016 Thompson Remove JIBB ######################################################################## @@ -74,6 +74,9 @@ if (($node =~ discover*) || ($node =~ borg*) || \ set site = "NCCS" + # NCCS now has both SLES15 and SLES12 machines + set OS_VERSION=`grep VERSION_ID /etc/os-release | cut -d= -f2 | cut -d. -f1 | sed 's/"//g'` + else if (($node =~ pfe*) || ($node =~ tfe*) || \ ($node =~ r[0-9]*i[0-9]*n[0-9]*) || \ ($node =~ r[0-9]*c[0-9]*t[0-9]*n[0-9]*)) then @@ -143,20 +146,31 @@ if ( $site == NCCS ) then set mod1 = GEOSenv - set mod2 = comp/gcc/10.1.0 - set mod3 = comp/intel/2021.2.0 + if ( $OS_VERSION == 12 ) then - set mod4 = mpi/impi/2021.2.0 + set mod2 = comp/gcc/10.1.0 + set mod3 = comp/intel/2021.2.0 + # Move to Intel MPI 2021.6 for cascade lake at NCCS + set mod4 = mpi/impi/2021.6.0 + set mod5 = python/GEOSpyD/Min4.11.0_py3.9_AND_Min4.8.3_py2.7 + set basedir = /discover/swdev/gmao_SIteam/Baselibs/ESMA-Baselibs-6.2.7/x86_64-pc-linux-gnu/ifort_2021.2.0-intelmpi_2021.2.0 + set usemod1 = /discover/swdev/gmao_SIteam/modulefiles-SLES12 - set mod5 = python/GEOSpyD/Min4.11.0_py3.9_AND_Min4.8.3_py2.7 + else - set basedir = /discover/swdev/gmao_SIteam/Baselibs/ESMA-Baselibs-6.2.7/x86_64-pc-linux-gnu/ifort_2021.2.0-intelmpi_2021.2.0 + set mod2 = comp/gcc/11.4.0 + set mod3 = comp/intel/2021.6.0 + set mod4 = mpi/impi/2021.13 + set mod5 = python/GEOSpyD/Min24.4.0-0_py3.11 + set basedir = /discover/swdev/gmao_SIteam/Baselibs/ESMA-Baselibs-6.2.7/x86_64-pc-linux-gnu/ifort_2021.6.0-intelmpi_2021.13.0-SLES15 + set usemod1 = /discover/swdev/gmao_SIteam/modulefiles-SLES15 + + endif set mods = ( $mod1 $mod2 $mod3 $mod4 $mod5 ) set modinit = /usr/share/modules/init/csh set loadmodules = 0 - set usemod1 = /discover/swdev/gmao_SIteam/modulefiles-SLES12 set usemods = ( $usemod1 ) set usemodules = 1 @@ -221,11 +235,11 @@ else if ( $site == GMAO.janus ) then # Bender # #===========# else if ( $site == ACDL.bender ) then - set basedir=/ford1/share/gmao_SIteam/Baselibs/GMAO-Baselibs-5_0_0/x86_64-pc-linux-gnu/ifort_13.1.1.163-openmpi_1.8.1 + set basedir=/ford1/share/gmao_SIteam/Baselibs/GMAO-Baselibs-5_0_0/x86_64-pc-linux-gnu/ifort_13.1.1.163-openmpi_1.8.1 set BASEBIN=$basedir/$arch/bin set PYBIN = /share/dasilva/epd/epd-7.3-2-rh5-x86_64/bin set IFCBIN = /opt/intel/composer_xe_2013.5.192/bin - set MPIBIN = /ford1/share/gmao_SIteam/MPI/openmpi-1.8.1-ifort-13.1.1.163/bin + set MPIBIN = /ford1/share/gmao_SIteam/MPI/openmpi-1.8.1-ifort-13.1.1.163/bin source $IFCBIN/ifortvars.csh intel64 setenv MKLPATH $MKLROOT/lib set path = ( $BASEBIN $MPIBIN $PYBIN $path ) @@ -456,7 +470,7 @@ DESCRIPTION add the BASEDIR lib directory to LD_LIBRARY_PATH (if necessary), and will load library modules when sourced. - If the script is called with "basedir", "modules", "modinit", or + If the script is called with "basedir", "modules", "modinit", or "loadmodules", then it will echo the values to standard output without modifying the environment. From c48576313603626117246b2a5e5cf9d1fe6a1877 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Fri, 24 Jan 2025 15:09:49 -0500 Subject: [PATCH 3/6] Add reservation option --- build.csh | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/build.csh b/build.csh index fc997b4..e6930ed 100755 --- a/build.csh +++ b/build.csh @@ -78,6 +78,7 @@ if (! $?proc) setenv proc "" if (! $?prompt) setenv prompt 1 if (! $?queue) setenv queue "" if (! $?partition) setenv partition "" +if (! $?reservation) setenv reservation "" if (! $?account) setenv account "" if (! $?tmpdir) setenv tmpdir "" if (! $?walltime) setenv walltime "" @@ -203,6 +204,13 @@ while ($#argv) setenv partition "--partition=$1" endif + # submit batch job to specified reservation + #---------------------------------------- + if ("$1" == "-reservation") then + shift; if (! $#argv) goto usage + setenv reservation "--reservation=$1" + endif + # submit batch job to specified account #-------------------------------------- if ("$1" == "-account") then @@ -511,6 +519,7 @@ if ($ddb) then echo "queue = $queue" if ($SITE == NCCS) then echo "partition = $partition" + echo "reservation = $reservation" echo "slurm_constraint = $slurm_constraint" endif echo "account = $account" @@ -792,14 +801,14 @@ else if ( $SITE == NCCS ) then # two OSs. For some reason, if you submit a Milan job from a SLES12 # headnode, it was seeing SLES12 module paths. We believe this is # because SLURM by default exports all the environment - sbatch $groupflag $partition $queue \ + sbatch $groupflag $partition $reservation $queue \ $slurm_constraint \ --job-name=$jobname \ --output=$jobname.o%j \ --nodes=1 \ --ntasks=${numjobs} \ --time=$walltime \ - --export ESMADIR=${ESMADIR},cmake_build_type=${cmake_build_type},EXTRA_CMAKE_FLAGS=${EXTRA_CMAKE_FLAGS},FORTRAN_COMPILER=${FORTRAN_COMPILER},INSTALL_SOURCE_TARFILE=${INSTALL_SOURCE_TARFILE},verbose=${verbose},GMI_MECHANISM_FLAG=${GMI_MECHANISM_FLAG},Pbuild_build_directory=${Pbuild_build_directory},Pbuild_install_directory=${Pbuild_install_directory},usegnu=${usegnu},notar=${notar},tmpdir=${tmpdir},docmake=${docmake},debug=${debug},aggressive=${aggressive},BUILDDIR_PASSED=${BUILDDIR_PASSED},INSTALLDIR_PASSED=${INSTALLDIR_PASSED},queue=${queue},partition=${partition},cleanFLAG=${cleanFLAG} \ + --export ESMADIR=${ESMADIR},cmake_build_type=${cmake_build_type},EXTRA_CMAKE_FLAGS=${EXTRA_CMAKE_FLAGS},FORTRAN_COMPILER=${FORTRAN_COMPILER},INSTALL_SOURCE_TARFILE=${INSTALL_SOURCE_TARFILE},verbose=${verbose},GMI_MECHANISM_FLAG=${GMI_MECHANISM_FLAG},Pbuild_build_directory=${Pbuild_build_directory},Pbuild_install_directory=${Pbuild_install_directory},usegnu=${usegnu},notar=${notar},tmpdir=${tmpdir},docmake=${docmake},debug=${debug},aggressive=${aggressive},BUILDDIR_PASSED=${BUILDDIR_PASSED},INSTALLDIR_PASSED=${INSTALLDIR_PASSED},queue=${queue},partition=${partition},reservation=${reservation},cleanFLAG=${cleanFLAG} \ $waitflag \ $0 unset echo @@ -903,6 +912,9 @@ endif if ("$partition" != "") then echo1 "partition: $partition" endif +if ("$reservation" != "") then + echo1 "reservation: $reservation" +endif if ("$account" != "") then echo1 "account: $account" echo1 "Pbuild_build_directory: $Pbuild_build_directory" @@ -988,6 +1000,8 @@ flagged options -i run interactively rather than queuing job -q qos/queue send batch job to qos/queue -partition partition send batch job to partition (in case SLURM queue not on default compute partition) + -reservation reservation + send batch job to reservation -account account send batch job to account -walltime hh:mm:ss time to use as batch walltime at job submittal From 67b5d4cb44945eaa1408d6ea7798c1106fa42189 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Fri, 24 Jan 2025 15:19:32 -0500 Subject: [PATCH 4/6] Add -cas15 option --- build.csh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/build.csh b/build.csh index e6930ed..8f6e442 100755 --- a/build.csh +++ b/build.csh @@ -131,6 +131,7 @@ while ($#argv) if ("$1" == "-mil") set nodeTYPE = "Milan" if ("$1" == "-rom") set nodeTYPE = "Rome" if ("$1" == "-cas") set nodeTYPE = "CascadeLake" + if ("$1" == "-cas15") set nodeTYPE = "15CascadeLake" if ("$1" == "-sky") set nodeTYPE = "Skylake" if ("$1" == "-bro") set nodeTYPE = "Broadwell" if ("$1" == "-has") set nodeTYPE = "Haswell" @@ -316,7 +317,7 @@ endif if ($SITE == NCCS) then set nT = `echo $nodeTYPE| tr "[A-Z]" "[a-z]" | cut -c1-3 ` - if (($nT != sky) && ($nT != cas) && ($nT != mil) && ($nT != any)) then + if (($nT != sky) && ($nT != cas) && ($nT != mil) && ($nT != 15c) && ($nT != any)) then echo "ERROR. Unknown node type at NCCS: $nodeTYPE" exit 1 endif @@ -326,11 +327,13 @@ if ($SITE == NCCS) then if ($nT == any) @ NCPUS_DFLT = 40 if ($nT == sky) @ NCPUS_DFLT = 40 if ($nT == cas) @ NCPUS_DFLT = 48 + if ($nT == 15c) @ NCPUS_DFLT = 48 if ($nT == mil) @ NCPUS_DFLT = 128 if ($nT == any) set proc = 'any' if ($nT == sky) set proc = 'sky' if ($nT == cas) set proc = 'cas' + if ($nT == 15c) set proc = 'cas' if ($nT == mil) set proc = 'mil' # If we are using GNU at NCCS, we can*only* use the cas or mil processors @@ -358,6 +361,10 @@ if ($SITE == NCCS) then set partition = '--partition=compute' endif + if ("$nT" == "15c") then + set reservation = "--reservation=sles15_cas" + endif + endif # at NAS @@ -456,7 +463,7 @@ if ($SITE == NCCS) then if ($oncompnode) then set OS_VERSION=`grep VERSION_ID /etc/os-release | cut -d= -f2 | cut -d. -f1 | sed 's/"//g'` else - if ($nT == mil) then + if ( ($nT == mil) || ($nT == 15c) ) then set OS_VERSION = 15 else set OS_VERSION = 12 From f189eabe3381a0d715db3ae2fd87598f0cb74b34 Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Mon, 17 Mar 2025 15:31:07 -0400 Subject: [PATCH 5/6] remove temporary cas15 stuff - all sles15 now --- build.csh | 106 +++++++++++------------------------------------------- 1 file changed, 20 insertions(+), 86 deletions(-) diff --git a/build.csh b/build.csh index 8f6e442..b64de25 100755 --- a/build.csh +++ b/build.csh @@ -78,7 +78,6 @@ if (! $?proc) setenv proc "" if (! $?prompt) setenv prompt 1 if (! $?queue) setenv queue "" if (! $?partition) setenv partition "" -if (! $?reservation) setenv reservation "" if (! $?account) setenv account "" if (! $?tmpdir) setenv tmpdir "" if (! $?walltime) setenv walltime "" @@ -131,7 +130,6 @@ while ($#argv) if ("$1" == "-mil") set nodeTYPE = "Milan" if ("$1" == "-rom") set nodeTYPE = "Rome" if ("$1" == "-cas") set nodeTYPE = "CascadeLake" - if ("$1" == "-cas15") set nodeTYPE = "15CascadeLake" if ("$1" == "-sky") set nodeTYPE = "Skylake" if ("$1" == "-bro") set nodeTYPE = "Broadwell" if ("$1" == "-has") set nodeTYPE = "Haswell" @@ -205,13 +203,6 @@ while ($#argv) setenv partition "--partition=$1" endif - # submit batch job to specified reservation - #---------------------------------------- - if ("$1" == "-reservation") then - shift; if (! $#argv) goto usage - setenv reservation "--reservation=$1" - endif - # submit batch job to specified account #-------------------------------------- if ("$1" == "-account") then @@ -308,8 +299,8 @@ endif # default nodeTYPE #----------------- if (! $?nodeTYPE) then - if ($SITE == NCCS) set nodeTYPE = "Any" - if ($SITE == NAS) set nodeTYPE = "Skylake" + if ($SITE == NCCS) set nodeTYPE = "Milan" + if ($SITE == NAS) set nodeTYPE = "Rome" endif # at NCCS @@ -317,38 +308,30 @@ endif if ($SITE == NCCS) then set nT = `echo $nodeTYPE| tr "[A-Z]" "[a-z]" | cut -c1-3 ` - if (($nT != sky) && ($nT != cas) && ($nT != mil) && ($nT != 15c) && ($nT != any)) then + if ( ($nT != cas) && ($nT != mil) && ($nT != any) ) then echo "ERROR. Unknown node type at NCCS: $nodeTYPE" exit 1 endif # For the any node, set the default to 40 cores as # this is the least number of cores you will get - if ($nT == any) @ NCPUS_DFLT = 40 - if ($nT == sky) @ NCPUS_DFLT = 40 + if ($nT == any) @ NCPUS_DFLT = 48 if ($nT == cas) @ NCPUS_DFLT = 48 - if ($nT == 15c) @ NCPUS_DFLT = 48 if ($nT == mil) @ NCPUS_DFLT = 128 if ($nT == any) set proc = 'any' - if ($nT == sky) set proc = 'sky' if ($nT == cas) set proc = 'cas' - if ($nT == 15c) set proc = 'cas' if ($nT == mil) set proc = 'mil' - # If we are using GNU at NCCS, we can*only* use the cas or mil processors - # as OpenMPI is only built for Infiniband - if ($usegnu) then - if ($nT == mil) then - echo "Using GNU at NCCS, setting queue to cas" - set proc = 'mil' - else - echo "Using GNU at NCCS, setting queue to cas" - set proc = 'cas' - endif - set slurm_constraint = "--constraint=$proc" + # Note if the user has not set proc, if we are + # building with GNU, we have to select something + # as the GNU has different answers. So we default + # to mil + if ($usegnu && ($nT == any)) then + echo "WARNING: Setting node type to mil as GNU optimization is processor dependent" + set slurm_constraint = "--constraint=mil" else if ($nT == any) then - set slurm_constraint = "--constraint=sky|cas" + set slurm_constraint = "--constraint=mil|cas" else set slurm_constraint = "--constraint=$proc" endif @@ -361,10 +344,6 @@ if ($SITE == NCCS) then set partition = '--partition=compute' endif - if ("$nT" == "15c") then - set reservation = "--reservation=sles15_cas" - endif - endif # at NAS @@ -372,11 +351,12 @@ endif if ( $SITE == NAS ) then set nT = `echo $nodeTYPE | cut -c1-3 | tr "[A-Z]" "[a-z]"` - if (($nT != has) && ($nT != bro) && ($nT != sky) && ($nT != cas) && ($nT != rom)) then + if (($nT != has) && ($nT != bro) && ($nT != sky) && ($nT != cas) && ($nT != rom) && ($nT != mil)) then echo "ERROR. Unknown node type at NAS: $nodeTYPE" exit 2 endif + if ($nT == mil) set nT = 'mil_ait' if ($nT == rom) set nT = 'rom_ait' if ($nT == sky) set nT = 'sky_ele' if ($nT == cas) set nT = 'cas_ait' @@ -387,6 +367,7 @@ if ( $SITE == NAS ) then if ($nT == sky_ele) @ NCPUS_DFLT = 40 if ($nT == cas_ait) @ NCPUS_DFLT = 40 if ($nT == rom_ait) @ NCPUS_DFLT = 128 + if ($nT == mil_ait) @ NCPUS_DFLT = 128 # TMPDIR needs to be reset #------------------------- @@ -446,42 +427,6 @@ if (! $?Pbuild_install_directory) then endif endif -# If we are at NCCS, because of the dual OSs, we decorate the build and -# install directory with the OS name. If we submit to Milan, we will add -# -SLES15, otherwise -SLES12 to the build and install directories. But, -# we only do this if the user has not specified a build directory or -# install directory -# --------------------------------------------------------------------- - -if ($SITE == NCCS) then - # We now have to handle this in two ways. One if we are on a compute node and one if we aren't. - # This is because of how this script works where it sort of submits itself to the batch system - # and many of the variables known by the script before submission are lost after submission. - # So if we are on a compute node, we detect the OS version directly, but if we are just submitting on a - # head node, we instead have to just use the processor type passed in. We'll use oncompnode to detect - # which case we are in. - if ($oncompnode) then - set OS_VERSION=`grep VERSION_ID /etc/os-release | cut -d= -f2 | cut -d. -f1 | sed 's/"//g'` - else - if ( ($nT == mil) || ($nT == 15c) ) then - set OS_VERSION = 15 - else - set OS_VERSION = 12 - endif - endif - # We also check if we already appended SLES - if (! $?BUILDDIR && "$BUILDDIR_PASSED" == "NO") then - if ($Pbuild_build_directory !~ "*-SLES${OS_VERSION}") then - setenv Pbuild_build_directory ${Pbuild_build_directory}-SLES${OS_VERSION} - endif - endif - if (! $?INSTALLDIR && "$INSTALLDIR_PASSED" == "NO") then - if ($Pbuild_install_directory !~ "*-SLES${OS_VERSION}") then - setenv Pbuild_install_directory ${Pbuild_install_directory}-SLES${OS_VERSION} - endif - endif -endif - if ("$FORTRAN_COMPILER" == "") then if ($usegnu) then setenv FORTRAN_COMPILER 'gfortran' @@ -526,7 +471,6 @@ if ($ddb) then echo "queue = $queue" if ($SITE == NCCS) then echo "partition = $partition" - echo "reservation = $reservation" echo "slurm_constraint = $slurm_constraint" endif echo "account = $account" @@ -804,18 +748,13 @@ else if ( $SITE == NAS ) then else if ( $SITE == NCCS ) then if ("$walltime" == "") setenv walltime "1:00:00" set echo - # NOTE: The weird long export line below is needed at NCCS because of the - # two OSs. For some reason, if you submit a Milan job from a SLES12 - # headnode, it was seeing SLES12 module paths. We believe this is - # because SLURM by default exports all the environment - sbatch $groupflag $partition $reservation $queue \ + sbatch $groupflag $partition $queue \ $slurm_constraint \ --job-name=$jobname \ --output=$jobname.o%j \ --nodes=1 \ --ntasks=${numjobs} \ --time=$walltime \ - --export ESMADIR=${ESMADIR},cmake_build_type=${cmake_build_type},EXTRA_CMAKE_FLAGS=${EXTRA_CMAKE_FLAGS},FORTRAN_COMPILER=${FORTRAN_COMPILER},INSTALL_SOURCE_TARFILE=${INSTALL_SOURCE_TARFILE},verbose=${verbose},GMI_MECHANISM_FLAG=${GMI_MECHANISM_FLAG},Pbuild_build_directory=${Pbuild_build_directory},Pbuild_install_directory=${Pbuild_install_directory},usegnu=${usegnu},notar=${notar},tmpdir=${tmpdir},docmake=${docmake},debug=${debug},aggressive=${aggressive},BUILDDIR_PASSED=${BUILDDIR_PASSED},INSTALLDIR_PASSED=${INSTALLDIR_PASSED},queue=${queue},partition=${partition},reservation=${reservation},cleanFLAG=${cleanFLAG} \ $waitflag \ $0 unset echo @@ -919,9 +858,6 @@ endif if ("$partition" != "") then echo1 "partition: $partition" endif -if ("$reservation" != "") then - echo1 "reservation: $reservation" -endif if ("$account" != "") then echo1 "account: $account" echo1 "Pbuild_build_directory: $Pbuild_build_directory" @@ -1007,18 +943,16 @@ flagged options -i run interactively rather than queuing job -q qos/queue send batch job to qos/queue -partition partition send batch job to partition (in case SLURM queue not on default compute partition) - -reservation reservation - send batch job to reservation -account account send batch job to account -walltime hh:mm:ss time to use as batch walltime at job submittal - -mil compile on Milan nodes (only at NCCS) - -rom compile on Rome nodes (only at NAS) + -mil compile on Milan nodes (default at NCCS) + -rom compile on Rome nodes (default at NAS, only at NAS) -cas compile on Cascade Lake nodes - -sky compile on Skylake nodes (default at NAS) + -sky compile on Skylake nodes (only at NAS) -bro compile on Broadwell nodes (only at NAS) -has compile on Haswell nodes (only at NAS) - -any compile on either Sky or Cascade Lake node (only at NCCS with SLURM, default at NCCS) + -any compile on any node (only at NCCS) extra cmake options From 6c218d469881834465b01743c94f146ee0bbfbf5 Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Mon, 17 Mar 2025 15:36:05 -0400 Subject: [PATCH 6/6] explain --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7124ea..23982cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed ### Added +## [3.4.0.3] - 2025-03-17 + +### Changed + +- Remove sles12 support; remove intermediate sles15 cas replace w/ simply cas + ## [3.4.0.2] - 2025-01-24 ### Changed