diff --git a/apps/spack/build_spack.sh b/apps/spack/build_spack.sh index c6ec9258d..d02db7e9a 100755 --- a/apps/spack/build_spack.sh +++ b/apps/spack/build_spack.sh @@ -2,7 +2,7 @@ set -e APPS_SPACK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" APP_NAME=spack -APP_VERSION=0.16.0 +APP_VERSION=0.16.3 SHARED_APP=${SHARED_APP:-/apps} INTEL_MPI_VERSION=${INTEL_MPI_VERSION:-2018.4.274} # This should be retrieved dynamically diff --git a/apps/spack/compilers.yaml b/apps/spack/compilers.yaml index 42cac4c45..b117ed82e 100644 --- a/apps/spack/compilers.yaml +++ b/apps/spack/compilers.yaml @@ -4,7 +4,7 @@ compilers: extra_rpaths: [] flags: {} modules: [] - operating_system: centos7 + operating_system: almalinux8 paths: cc: /usr/bin/gcc cxx: /usr/bin/g++ @@ -17,7 +17,7 @@ compilers: extra_rpaths: [] flags: {} modules: [gcc-9.2.0] - operating_system: centos7 + operating_system: almalinux8 paths: cc: /opt/gcc-9.2.0/bin/gcc cxx: /opt/gcc-9.2.0/bin/g++ diff --git a/apps/spack/packages.yaml b/apps/spack/packages.yaml index ded47454b..89015ee12 100644 --- a/apps/spack/packages.yaml +++ b/apps/spack/packages.yaml @@ -2,7 +2,7 @@ packages: openmpi: externals: - - spec: openmpi@4.0.5%gcc@9.2.0 + - spec: openmpi@5.0.2%gcc@9.2.0 modules: - mpi/openmpi buildable: False diff --git a/apps/wrf/automation/build-folder-structure.sh b/apps/wrf/automation/build-folder-structure.sh new file mode 100644 index 000000000..8354d1f6a --- /dev/null +++ b/apps/wrf/automation/build-folder-structure.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +WRFDAT="/data/wrfdata" + +echo "creating ${WRFDAT}/wpsdir/" +mkdir -p ${WRFDAT}/wpsdir/ + +echo "creating ${WRFDAT}/wrfdir/" +mkdir -p ${WRFDAT}/wrfdir/ + +echo "creating ${WRFDAT}/tables/wps/" +mkdir -p ${WRFDAT}/tables/wps/ + +echo "creating ${WRFDAT}/tables/wrf/" +mkdir -p ${WRFDAT}/tables/wrf/ + +echo "creating ${WRFDAT}/tables/namelist/" +mkdir -p ${WRFDAT}/tables/namelist/ + +echo "creating ${WRFDAT}/gfs_data/" +mkdir -p ${WRFDAT}/gfs_data/ + +echo "creating ${WRFDAT}/geog/" +mkdir -p ${WRFDAT}/geog/ + +echo "creating /apps/scripts/" +mkdir -p /apps/scripts/ + +echo "copying namelist*.wps" +cp -f /data/azurehpc/apps/wrf/automation/namelist*.wps ${WRFDAT}/tables/namelist/ + +echo "copying namelist*.input" +cp -f /data/azurehpc/apps/wrf/automation/namelist*.input ${WRFDAT}/tables/namelist/ + +echo "creating link to scripts" +#cd /data/azurehpc/apps/wrf/automation/ +#cp /data/azurehpc/apps/wrf/automation/fwddatan.awk /apps/scripts/ +#cp /data/azurehpc/apps/wrf/automation/get_gfs_data.py /apps/scripts/ +#cp run_*.slurm run_*.pbs submit*.sh /apps/scripts/ +ln -s /data/azurehpc/apps/wrf/automation/* /apps/scripts/ + +echo "creating links to WPF files" +ln -s /apps/hbv3/wps-openmpi/WPS-4.1/* /data/wrfdata/tables/wps/ + +echo "creating links to WRF files" +ln -s /apps/hbv3/wrf-openmpi/WRF-4.1.5/run/* /data/wrfdata/tables/wrf/ + +echo "Moving geog files" +#mv albedo_modis albedo_modis.tar.bz2 greenfrac_fpar_modis greenfrac_fpar_modis.tar.bz2 lai_modis_10m lai_modis_10m.tar.bz2 maxsnowalb_modis maxsnowalb_modis.tar.bz2 modis_landuse_20class_30s_with_lakes modis_landuse_20class_30s_with_lakes.tar.bz2 orogwd_10m orogwd_10m.tar.bz2 soiltemp_1deg soiltemp_1deg.tar.bz2 soiltype_bot_30s soiltype_bot_30s.tar.bz2 soiltype_top_30s soiltype_top_30s.tar.bz2 topo_10m topo_10m.tar.bz2 topo_2m topo_2m.tar.bz2 topo_30s topo_30s.tar.bz2 topo_gmted2010_30s topo_gmted2010_30s.tar.bz2 geog + diff --git a/apps/wrf/automation/fwddatan.awk b/apps/wrf/automation/fwddatan.awk new file mode 100644 index 000000000..427b44163 --- /dev/null +++ b/apps/wrf/automation/fwddatan.awk @@ -0,0 +1,88 @@ +{ data = $1; + hplus = $2; + len = length(data); + res = len - 6; + dias_mes = "31,28,31,30,31,30,31,31,30,31,30,31" + ano = substr(data,1,4); + mes = substr(data,5,2); + dia = substr(data,7,2); + resto = substr(data,9,res); + fs="," + a=split(dias_mes,dias,fs); +# an1 = (1900+ano)/4.0; + an1 = ano/4.0; + bix = substr(an1,1,3); + + if (bix == an1){ + dias_mes = "31,29,31,30,31,30,31,31,30,31,30,31"; + split(dias_mes,dias,fs);} + + diao = dia + if(diao<10) { diao= substr(data,8,1) + 1 - 1}; + + hres = resto + hplus; + ndia = hres/24.0; + diai = int (ndia) + + if(diai >= 1) + { + hres = hres - (diai*24) + diao = dia + diai; + }; + + if(mes<10) {mes = substr(data,6,1) + 1 - 1}; + + if(diao > dias[mes]) + { + diao = diao - dias[mes]; + mes=mes+1; + if(mes > 12) + { + mes=1; + anoteste=ano; + ano = ano +1; + if(ano==100) + { + ano="0"; + }; + + }; + }; + + + if(hres < 0) + { + diai = diai - 1 + hres = hres - (diai*24) + if(hres == 24) + { + hres=0; + diai = diai + 1; + }; + diao = dia + diai; + }; + + if(diao<1) + { + mes=mes-1; + + if(mes<1) + { + mes=12; + ano = ano -1; + if(ano<0) + { + ano=99; + }; + }; + diao = dias[mes] + diao ; + } + + + if(diao<10) {diao="0"diao}; + if(ano<10) {ano="0"ano}; + if(mes<10) {mes="0"mes}; + if(hres<10) {hres="0"hres}; + + print ano mes diao hres; +} diff --git a/apps/wrf/automation/get_gfs_data.py b/apps/wrf/automation/get_gfs_data.py new file mode 100644 index 000000000..5549238ab --- /dev/null +++ b/apps/wrf/automation/get_gfs_data.py @@ -0,0 +1,73 @@ +#from wget import download +import wget +import datetime as dt +import os, subprocess +import sys +from time import time as timer +from multiprocessing.pool import ThreadPool +#import requests +year = int(sys.argv[1]) +month = int(sys.argv[2]) +day = int(sys.argv[3]) + +##dia = dt.date.today() +#dia = dt.datetime(dia.year,dia.month,dia.day,12) +dia = dt.datetime(year,month,day,12) +print(dia) +inicio = 00 +##fim = 28 ## Dados para 1 dia +fim = 135 ## Dados para 5 dias + +data_sys = dia.strftime("%Y%m%d%H") +data_gfs = dia.strftime("%Y%m%d") + +##path_home = "/data/wrfdata/itv-data/" +path_home = "/data/wrfdata/gfs_data/" + + +#### CASO O DIRETORIO PARA O DOWNLOAD NAO EXISTE, ELE SERA CRIADO #### + +if not os.path.isdir(path_home + "/" + data_sys ): + os.system("mkdir -p " + path_home + data_sys ) + +#### URL (http://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod) +#### Alteracao realizada em 10 / set / 2020 +#### URL (ftp://ftpprd.ncep.noaa.gov/pub/data/nccf/com/gfs/prod) + +#### DOWNLOAD DOS ARQUIVOS ##### +args_inputs = [] +j=0 +for i in range(inicio,fim,3): + k = str(i) + if i < 10 : + k = ('00' + k) + if i >= 10 and i < 100 : + k = ('0' + k) + + gfs = "gfs.t12z.pgrb2.0p25.f" + k + print(gfs) + + if not os.path.isfile(path_home + "/" + data_sys + "/" + gfs): + f_out = path_home + "/" + data_sys + '/' + gfs + +#### url = ('http://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.' + data_gfs + '/12/' + gfs) +#### url = ('ftp://ftpprd.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.' + data_gfs + '/12/' + gfs) +#### url = ('ftp://ftpprd.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.' + data_gfs + '/12/atmos/' + gfs) + url = ('https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.' + data_gfs + '/12/atmos/' + gfs) + + args_inputs.append ((f_out, url)) + j=j+1 + +def fetch_url(entry): + path, uri = entry + print ("Downloading...", path) + filename = wget.download(uri,path_home + data_sys) + return path + +start=timer() +results = ThreadPool(6).imap_unordered(fetch_url, args_inputs) + +for path in results: + print(path) + +print("Elapsed Time: ", timer() - start) diff --git a/apps/wrf/automation/namelist-new.input b/apps/wrf/automation/namelist-new.input new file mode 100644 index 000000000..482442b46 --- /dev/null +++ b/apps/wrf/automation/namelist-new.input @@ -0,0 +1,140 @@ +&time_control + run_days = 0, + run_hours = 132, + run_minutes = 0, + run_seconds = 0, + start_year = aai, aai, + start_month = mmi, mmi, + start_day = ddi, ddi, + start_hour = hhi, hhi, + end_year = aaf, aaf, + end_month = mmf, mmf, + end_day = ddf, ddf, + end_hour = hhf, hhf, + interval_seconds = 10800, + input_from_file = .true.,.true., + history_interval = 60, 60, + frames_per_outfile = 1, 1, + restart = .false., + restart_interval = 7200, + io_form_history = 2, + io_form_restart = 2, + io_form_input = 2, + io_form_boundary = 2, + debug_level = 0, + + / + + &domains +!time_step = 100, + time_step = 80, + time_step_fract_num = 0, + time_step_fract_den = 1, + max_dom = 2, + s_we = 1, 1, + e_we = 420, 982, + s_sn = 1, 1, + e_sn = 300, 646, + p_top_requested = 7200, + num_metgrid_levels = 34, + num_metgrid_soil_levels = 4, + use_surface = .true., + sfcp_to_sfcp = .true., + dx = 9000., 3000., + dy = 9000., 3000., + grid_id = 1, 2, + parent_id = 1, 1, + i_parent_start = 1, 46, + j_parent_start = 1, 39, + parent_grid_ratio = 1, 3, + parent_time_step_ratio = 1, 3, + feedback = 1, + smooth_option = 0, + / + + &physics + mp_physics = 95, 95, + ra_lw_physics = 4, 4, + ra_sw_physics = 4, 4, + slope_rad = 1, 1, + topo_shading = 0, 1, + radt = 30, 30, + sf_urban_physics = 1, 1, + sf_sfclay_physics = 1, 1, + sf_surface_physics = 2, 2, + bl_pbl_physics = 1, 1, + bldt = 0, 0, + cu_physics = 3, 3, + cudt = 5, 5, + cugd_avedx = 3, + sst_update = 0, + isfflx = 1, + ifsnow = 0, + icloud = 1, + surface_input_source = 1, + num_soil_layers = 4, + maxiens = 1, + maxens = 3, + maxens2 = 3, + maxens3 = 16, + ensdim = 144, + + / + + &fdda + / + + &dynamics + dyn_opt = 2, + rk_ord = 3, + w_damping = 1, + diff_opt = 1, 1, + km_opt = 4, 4, + damp_opt = 3, + base_temp = 290., + zdamp = 5000., 5000., + dampcoef = 0.2, 0.2, + khdif = 0, 0, + kvdif = 0, 0, + smdiv = 0.1, 0.1, + emdiv = 0.01, 0.01, + epssm = 0.1, 0.1, + non_hydrostatic = .true., .true., + time_step_sound = 0, 0, + h_mom_adv_order = 5, 5, + v_mom_adv_order = 3, 3, + h_sca_adv_order = 3, 5, + v_sca_adv_order = 2, 3, + moist_adv_opt = 1, 1, + scalar_adv_opt = 1, 1, + chem_adv_opt = 1, 1, + tke_adv_opt = 1, 1, + diff_6th_opt = 1, 1, + diff_6th_factor = 0.12, 0.12, + / + + &bdy_control + spec_bdy_width = 5, + spec_zone = 1, + relax_zone = 4, + specified = .true., .false., + periodic_x = .false., .false., + symmetric_xs = .false., .false., + symmetric_xe = .false., .false., + open_xs = .false., .false., + open_xe = .false., .false., + periodic_y = .false., .false., + symmetric_ys = .false., .false., + symmetric_ye = .false., .false., + open_ys = .false., .false., + open_ye = .false., .false., + nested = .false., .true., + / + + &grib2 + / + + &namelist_quilt + nio_tasks_per_group = 0, + nio_groups = 1, + / diff --git a/apps/wrf/automation/namelist-new.wps b/apps/wrf/automation/namelist-new.wps new file mode 100644 index 000000000..77c566019 --- /dev/null +++ b/apps/wrf/automation/namelist-new.wps @@ -0,0 +1,50 @@ +&share + wrf_core = 'ARW', + max_dom = 2, + start_date = 'aai-mmi-ddi_hhi:00:00','aai-mmi-ddi_hhi:00:00', + end_date = 'aaf-mmf-ddf_hhf:00:00','aaf-mmf-ddf_hhf:00:00', + interval_seconds = 10800 + io_form_geogrid = 2, + opt_output_from_geogrid_path = '/data/wrfdata/wpsdir/wrkday', + debug_level = 0, +/ + +&geogrid + parent_id = 1,1, + parent_grid_ratio = 1,3, + i_parent_start = 1,46, + j_parent_start = 1,39, + e_we = 420,982, + e_sn = 300,646, + geog_data_res = '30s','30s', + dx = 9000, + dy = 9000, + map_proj = 'lambert', + ref_lat = -3.0, + ref_lon = -48.0, + truelat1 = -3.0, + truelat2 = -3.0, + stand_lon = -48.0, + geog_data_path = '/data/wrfdata/geog', + opt_geogrid_tbl_path = '/data/wrfdata/wpsdir/wrkday/geogrid', + ref_x = 210.0, + ref_y = 150.0, +/ + +&ungrib + out_format = 'WPS', + prefix = 'FILE', +/ + +&metgrid + fg_name = 'FILE' + io_form_metgrid = 2, + opt_output_from_metgrid_path = '/data/wrfdata/wpsdir/wrkday', + opt_metgrid_tbl_path = '/data/wrfdata/wpsdir/wrkday/metgrid', +/ + +&mod_levs + press_pa = 201300, 200100, 100000, 95000, 90000, 85000, 80000, 75000, + 70000, 65000, 60000, 55000, 50000, 45000, 40000, 35000, + 30000, 25000, 20000, 15000, 10000, 5000, 1000 +/ diff --git a/apps/wrf/automation/namelist-new_conus2.5km-orig.input b/apps/wrf/automation/namelist-new_conus2.5km-orig.input new file mode 100644 index 000000000..8cb2219df --- /dev/null +++ b/apps/wrf/automation/namelist-new_conus2.5km-orig.input @@ -0,0 +1,87 @@ + &time_control + run_days = 0, + run_hours = 6, + run_minutes = 0, + run_seconds = 0, + start_year = 2018, + start_month = 06, + start_day = 17, + start_hour = 00, + start_minute = 00, + start_second = 00, + end_year = 2018, + end_month = 06, + end_day = 17, + end_hour = 12, + end_minute = 00, + end_second = 00, + interval_seconds = 10800, + input_from_file = .true., + history_interval = 180, + frames_per_outfile = 5, + restart = .false., + restart_interval = 360, + io_form_history = 2, + io_form_restart = 2, + io_form_input = 2, + io_form_boundary = 2, + debug_level = 0 + / + + &domains + time_step = 15, + time_step_fract_num = 0, + time_step_fract_den = 1, + max_dom = 1, + e_we = 1901, + e_sn = 1301, + e_vert = 35, + dx = 2500, + dy = 2500, + p_top_requested = 5000, + num_metgrid_levels = 32, + num_metgrid_soil_levels = 4, + grid_id = 1, + parent_id = 0, + i_parent_start = 1, + j_parent_start = 1, + parent_grid_ratio = 1, + parent_time_step_ratio = 1, + feedback = 1, + smooth_option = 0 + / + + &physics + physics_suite = 'CONUS' + cu_physics = 0, + radt = 10, + bldt = 0, + cudt = 5, + icloud = 1, + isfflx = 1, + isftcflx = 2, + num_soil_layers = 4, + num_land_cat = 21, + / + + &dynamics + w_damping = 1, + diff_opt = 1, + km_opt = 4, + khdif = 0, + kvdif = 0, + non_hydrostatic = .true., + / + + &bdy_control + spec_bdy_width = 5, + spec_zone = 1, + relax_zone = 4, + specified = .true., + nested = .false., + / + + &namelist_quilt + nio_tasks_per_group = 0, + nio_groups = 1, + / diff --git a/apps/wrf/automation/namelist-new_conus2.5km-orig.wps b/apps/wrf/automation/namelist-new_conus2.5km-orig.wps new file mode 100644 index 000000000..fe2c09c44 --- /dev/null +++ b/apps/wrf/automation/namelist-new_conus2.5km-orig.wps @@ -0,0 +1,56 @@ +&share + wrf_core = 'ARW', + max_dom = 1, + start_date = '2018-06-17_00:00:00', + end_date = '2018-06-17_12:00:00', + interval_seconds = 10800 + io_form_geogrid = 2, + opt_output_from_geogrid_path = '/data/wrfdata/wpsdir/wrkday', +/ + +&geogrid + parent_id = 1, + parent_grid_ratio = 1, + i_parent_start = 1, + j_parent_start = 1, + e_we = 1901, + e_sn = 1301, + ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!! IMPORTANT NOTE !!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! The default datasets used to produce the HGT_M, GREENFRAC, + ! and LU_INDEX/LANDUSEF fields have changed in WPS v3.8. The HGT_M field + ! is now interpolated from 30-arc-second USGS GMTED2010, the GREENFRAC + ! field is interpolated from MODIS FPAR, and the LU_INDEX/LANDUSEF fields + ! are interpolated from 21-class MODIS. + ! + ! To match the output given by the default namelist.wps in WPS v3.7.1, + ! the following setting for geog_data_res may be used: + ! + ! geog_data_res = 'gtopo_10m+usgs_10m+nesdis_greenfrac+10m','gtopo_2m+usgs_2m+nesdis_greenfrac+2m', + ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!! IMPORTANT NOTE !!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! + geog_data_res = 'default', + dx = 2500, + dy = 2500, + map_proj = 'lambert', + ref_lat = 40.0, + ref_lon = -98.0, + truelat1 = 30.0, + truelat2 = 60.0, + stand_lon = -98.0, + geog_data_path = '/data/wrfdata/geog', + opt_geogrid_tbl_path = '/data/wrfdata/wpsdir/wrkday/geogrid', +/ + +&ungrib + out_format = 'WPS', + prefix = 'FILE', +/ + +&metgrid + fg_name = 'FILE' + io_form_metgrid = 2, + opt_output_from_metgrid_path = '/data/wrfdata/wpsdir/wrkday', + opt_metgrid_tbl_path = '/data/wrfdata/wpsdir/wrkday/metgrid', +/ diff --git a/apps/wrf/automation/namelist-new_conus2.5km.input b/apps/wrf/automation/namelist-new_conus2.5km.input new file mode 100644 index 000000000..32ad7d690 --- /dev/null +++ b/apps/wrf/automation/namelist-new_conus2.5km.input @@ -0,0 +1,87 @@ + &time_control + run_days = 0, + run_hours = 6, + run_minutes = 0, + run_seconds = 0, + start_year = aai, + start_month = mmi, + start_day = ddi, + start_hour = hhi, + start_minute = 00, + start_second = 00, + end_year = aaf, + end_month = mmf, + end_day = ddf, + end_hour = hhf, + end_minute = 00, + end_second = 00, + interval_seconds = 10800, + input_from_file = .true., + history_interval = 180, + frames_per_outfile = 5, + restart = .false., + restart_interval = 360, + io_form_history = 2, + io_form_restart = 2, + io_form_input = 2, + io_form_boundary = 2, + debug_level = 0 + / + + &domains + time_step = 15, + time_step_fract_num = 0, + time_step_fract_den = 1, + max_dom = 1, + e_we = 1901, + e_sn = 1301, + e_vert = 35, + dx = 2500, + dy = 2500, + p_top_requested = 5000, + num_metgrid_levels = 32, + num_metgrid_soil_levels = 4, + grid_id = 1, + parent_id = 0, + i_parent_start = 1, + j_parent_start = 1, + parent_grid_ratio = 1, + parent_time_step_ratio = 1, + feedback = 1, + smooth_option = 0 + / + + &physics + physics_suite = 'CONUS' + cu_physics = 0, + radt = 10, + bldt = 0, + cudt = 5, + icloud = 1, + isfflx = 1, + isftcflx = 2, + num_soil_layers = 4, + num_land_cat = 21, + / + + &dynamics + w_damping = 1, + diff_opt = 1, + km_opt = 4, + khdif = 0, + kvdif = 0, + non_hydrostatic = .true., + / + + &bdy_control + spec_bdy_width = 5, + spec_zone = 1, + relax_zone = 4, + specified = .true., + nested = .false., + / + + &namelist_quilt + nio_tasks_per_group = 0, + nio_groups = 1, + / diff --git a/apps/wrf/automation/namelist-new_conus2.5km.wps b/apps/wrf/automation/namelist-new_conus2.5km.wps new file mode 100644 index 000000000..fd892da78 --- /dev/null +++ b/apps/wrf/automation/namelist-new_conus2.5km.wps @@ -0,0 +1,56 @@ +&share + wrf_core = 'ARW', + max_dom = 1, + start_date = 'aai-mmi-ddi_hhi:00:00', + end_date = 'aaf-mmf-ddf_hhf:00:00', + interval_seconds = 10800 + io_form_geogrid = 2, + opt_output_from_geogrid_path = '/data/wrfdata/wpsdir/wrkday', +/ + +&geogrid + parent_id = 1, + parent_grid_ratio = 1, + i_parent_start = 1, + j_parent_start = 1, + e_we = 1901, + e_sn = 1301, + ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!! IMPORTANT NOTE !!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! The default datasets used to produce the HGT_M, GREENFRAC, + ! and LU_INDEX/LANDUSEF fields have changed in WPS v3.8. The HGT_M field + ! is now interpolated from 30-arc-second USGS GMTED2010, the GREENFRAC + ! field is interpolated from MODIS FPAR, and the LU_INDEX/LANDUSEF fields + ! are interpolated from 21-class MODIS. + ! + ! To match the output given by the default namelist.wps in WPS v3.7.1, + ! the following setting for geog_data_res may be used: + ! + ! geog_data_res = 'gtopo_10m+usgs_10m+nesdis_greenfrac+10m','gtopo_2m+usgs_2m+nesdis_greenfrac+2m', + ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!! IMPORTANT NOTE !!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! + geog_data_res = 'default', + dx = 2500, + dy = 2500, + map_proj = 'lambert', + ref_lat = 40.0, + ref_lon = -98.0, + truelat1 = 30.0, + truelat2 = 60.0, + stand_lon = -98.0, + geog_data_path = '/data/wrfdata/geog', + opt_geogrid_tbl_path = '/data/wrfdata/wpsdir/wrkday/geogrid', +/ + +&ungrib + out_format = 'WPS', + prefix = 'FILE', +/ + +&metgrid + fg_name = 'FILE' + io_form_metgrid = 2, + opt_output_from_metgrid_path = '/data/wrfdata/wpsdir/wrkday', + opt_metgrid_tbl_path = '/data/wrfdata/wpsdir/wrkday/metgrid', +/ diff --git a/apps/wrf/automation/namelist.ARWpost b/apps/wrf/automation/namelist.ARWpost new file mode 100644 index 000000000..59e791ed0 --- /dev/null +++ b/apps/wrf/automation/namelist.ARWpost @@ -0,0 +1,24 @@ +&datetime + start_date = '2022-12-20_12:00:00', + end_date = '2022-12-20_14:00:00', +! end_date = '2022-12-26_00:00:00', + interval_seconds = 3600, + tacc = 0, + debug_level = 0, +/ + +&io + input_root_name = '/data/wrfdata/wrfdir/2022122012/wrfout_d02' + output_root_name = '/data/wrfdata/arwdir/2022122012/arwout_d02' + + mercator_defs = .true., + split_output = .true., + frames_per_outfile = 1, + plot = 'list', + fields = 'T2,U10,V10,RAINC,RAINNC,RAINSH', +/ + +&interp + interp_method = 0, + interp_levels = 1000.,950.,850.,700.,500.,400.,200., +/ diff --git a/apps/wrf/automation/run_wps1_openmpi.pbs b/apps/wrf/automation/run_wps1_openmpi.pbs new file mode 100644 index 000000000..0d31b11dc --- /dev/null +++ b/apps/wrf/automation/run_wps1_openmpi.pbs @@ -0,0 +1,71 @@ +#!/bin/bash +SKU_TYPE=${1:-$SKU_TYPE} +SKU_TYPE=${SKU_TYPE:-"hbv2"} + +path_scr=/apps/scripts +WRFDAT="/data/wrfdata" +WPSDIR=${WRFDAT}/wpsdir +WRFDIR=${WRFDAT}/wrfdir +WRKDAY=${2:-$WRKDAY} + +ANO=`echo ${WRKDAY} | cut -c 1-4` +MES=`echo ${WRKDAY} | cut -c 5-6` +DIA=`echo ${WRKDAY} | cut -c 7-8` +HOR=`echo ${WRKDAY} | cut -c 9-10` + +#DIAFIM=`date +%Y%m%d --date="$ANO$MES$DIA +5 day"` +DIAFIM=`echo ${WRKDAY} 132 | awk -f $path_scr/fwddatan.awk` + +ANOF=`echo ${DIAFIM} | cut -c 1-4` +MESF=`echo ${DIAFIM} | cut -c 5-6` +DIAF=`echo ${DIAFIM} | cut -c 7-8` +HORF=`echo ${DIAFIM} | cut -c 9-10` + + +echo $WRFDAT $WPSDIR $WRFDIR $WRKDAY $DIAFIM + +mkdir -p $WPSDIR/$WRKDAY + +#echo "INPUTDIR:" $INPUTDIR +echo "SKU_TYPE:" $SKU_TYPE + +SHARED_APP=${SHARED_APP:-/apps} + +echo "source envs" +source /data/azurehpc/apps/wrf/env-variables + +# Waintig 30 seconds +sleep 30s + +python3 ${path_scr}/get_gfs_data.py $ANO $MES $DIA + +NPROCS=`cat $PBS_NODEFILE | wc -l` +echo "NPROCS:"$NPROCS + +cd ${WPSDIR}/${WRKDAY} +echo "cd $WPSDIR/$WRKDAY" + +ln -s ${WRFDAT}/tables/wps/* $WPSDIR/$WRKDAY + +cp ${WRFDAT}/tables/namelist/namelist.wps $WPSDIR/$WRKDAY + +sed -i s/wrkday/${WRKDAY}/g namelist.wps + +sed -i s/aai/${ANO}/g namelist.wps +sed -i s/mmi/${MES}/g namelist.wps +sed -i s/ddi/${DIA}/g namelist.wps +sed -i s/hhi/${HOR}/g namelist.wps + +sed -i s/aaf/${ANOF}/g namelist.wps +sed -i s/mmf/${MESF}/g namelist.wps +sed -i s/ddf/${DIAF}/g namelist.wps +sed -i s/hhf/${HORF}/g namelist.wps + +mpi_options="-x LD_LIBRARY_PATH " +if [ -n $LD_PRELOAD ]; then + mpi_options+="-x LD_PRELOAD" +fi + +./link_grib.csh ${WRFDAT}/gfs_data/${WRKDAY}/gfs* +./ungrib.exe >& ungrib.log + diff --git a/apps/wrf/automation/run_wps1_openmpi.slurm b/apps/wrf/automation/run_wps1_openmpi.slurm new file mode 100644 index 000000000..04ce38144 --- /dev/null +++ b/apps/wrf/automation/run_wps1_openmpi.slurm @@ -0,0 +1,89 @@ +#!/bin/bash +#SBATCH --job-name=wps1 +#SBATCH --partition=hpc +#SBATCH --ntasks-per-node=16 +#SBATCH --nodes=1 +#SBATCH --exclusive +#SBATCH --output=slurm-wps1-%j.out + +# Capture start time +start_time=$(date +%s) + +SKU_TYPE=${1:-$SKU_TYPE} +SKU_TYPE=${SKU_TYPE:-"hbv3"} + +path_scr=/apps/scripts +WRFDAT="/data/wrfdata" +WPSDIR=${WRFDAT}/wpsdir +WRFDIR=${WRFDAT}/wrfdir +WRKDAY=${2:-$WRKDAY} +echo "WRKDAY:" $WRKDAY + +ANO=`echo ${WRKDAY} | cut -c 1-4` +MES=`echo ${WRKDAY} | cut -c 5-6` +DIA=`echo ${WRKDAY} | cut -c 7-8` +HOR=`echo ${WRKDAY} | cut -c 9-10` + +#DIAFIM=`date +%Y%m%d --date="$ANO$MES$DIA +5 day"` +DIAFIM=`echo ${WRKDAY} 132 | awk -f $path_scr/fwddatan.awk` + +ANOF=`echo ${DIAFIM} | cut -c 1-4` +MESF=`echo ${DIAFIM} | cut -c 5-6` +DIAF=`echo ${DIAFIM} | cut -c 7-8` +HORF=`echo ${DIAFIM} | cut -c 9-10` + +echo $WRFDAT $WPSDIR $WRFDIR $WRKDAY $DIAFIM + +mkdir -p $WPSDIR/$WRKDAY + +echo "SKU_TYPE:" $SKU_TYPE + +SHARED_APP=${SHARED_APP:-/apps} + +# Load environment variables +echo "Load environment variables" +source /data/azurehpc/apps/wrf/env-variables $SKU_TYPE + +# Waiting 30 seconds +echo "Waiting 30 seconds" +sleep 30s + +# Download gfs data +echo "Downloading gfs data..." +python3 ${path_scr}/get_gfs_data.py $ANO $MES $DIA + +echo "SLURM_SUBMIT_DIR:" $SLURM_SUBMIT_DIR + +cd ${WPSDIR}/${WRKDAY} +echo "cd $WPSDIR/$WRKDAY" + +echo "ln -s ${WRFDAT}/tables/wps/* $WPSDIR/$WRKDAY" +ln -s ${WRFDAT}/tables/wps/* $WPSDIR/$WRKDAY + +echo "Copying namelist.wps" +cp ${WRFDAT}/tables/namelist/namelist.wps $WPSDIR/$WRKDAY + +sed -i s/wrkday/${WRKDAY}/g namelist.wps + +sed -i s/aai/${ANO}/g namelist.wps +sed -i s/mmi/${MES}/g namelist.wps +sed -i s/ddi/${DIA}/g namelist.wps +sed -i s/hhi/${HOR}/g namelist.wps + +sed -i s/aaf/${ANOF}/g namelist.wps +sed -i s/mmf/${MESF}/g namelist.wps +sed -i s/ddf/${DIAF}/g namelist.wps +sed -i s/hhf/${HORF}/g namelist.wps + +# Capture end time +end_time=$(date +%s) +# Calculate duration +duration=$((end_time - start_time)) +# Format duration to hh:mm:ss +hours=$(printf "%02d" $((duration / 3600))) +minutes=$(printf "%02d" $(( (duration % 3600) / 60 ))) +seconds=$(printf "%02d" $((duration % 60))) +duration="${hours}:${minutes}:${seconds}" + +echo "WPS1 finished." +echo "Execution time: $duration seconds" \ No newline at end of file diff --git a/apps/wrf/automation/run_wps2_openmpi.pbs b/apps/wrf/automation/run_wps2_openmpi.pbs new file mode 100644 index 000000000..40e49f4c3 --- /dev/null +++ b/apps/wrf/automation/run_wps2_openmpi.pbs @@ -0,0 +1,30 @@ +#!/bin/bash +SKU_TYPE=${1:-$SKU_TYPE} +SKU_TYPE=${SKU_TYPE:-"hbv2"} + +WRFDAT="/data/wrfdata" +WPSDIR=${WRFDAT}/wpsdir +WRFDIR=${WRFDAT}/wrfdir +WRKDAY=${2:-$WRKDAY} + +echo $WRFDAT $WPSDIR $WRFDIR $WRKDAY + +echo "SKU_TYPE:" $SKU_TYPE + +echo "source envs" +source /data/azurehpc/apps/wrf/env-variables + +NPROCS=`cat $PBS_NODEFILE | wc -l` +echo "NPROCS:"$NPROCS + +cd $WPSDIR/$WRKDAY +echo "cd $WPSDIR/$WRKDAY" + +mpi_options="-x LD_LIBRARY_PATH " +if [ -n $LD_PRELOAD ]; then + mpi_options+="-x LD_PRELOAD" +fi +echo "PBS_NODEFILE:" $PBS_NODEFILE + +mpirun $mpi_options -n $NPROCS --hostfile $PBS_NODEFILE --bind-to numa geogrid.exe +mpirun $mpi_options -n $NPROCS --hostfile $PBS_NODEFILE --bind-to numa metgrid.exe diff --git a/apps/wrf/automation/run_wps2_openmpi.slurm b/apps/wrf/automation/run_wps2_openmpi.slurm new file mode 100644 index 000000000..c9153e982 --- /dev/null +++ b/apps/wrf/automation/run_wps2_openmpi.slurm @@ -0,0 +1,69 @@ +#!/bin/bash +#SBATCH --job-name=wps2 +#SBATCH --partition=hpc +#SBATCH --ntasks-per-node=16 +#SBATCH --nodes=1 +#SBATCH --exclusive +#SBATCH --output=slurm-wps2-%j.out + +# Capture start time +start_time=$(date +%s) + +SKU_TYPE=${1:-$SKU_TYPE} +SKU_TYPE=${SKU_TYPE:-"hbv3"} + +path_scr=/apps/scripts +WRFDAT="/data/wrfdata" +WPSDIR=${WRFDAT}/wpsdir +WRFDIR=${WRFDAT}/wrfdir +WRKDAY=${2:-$WRKDAY} +echo "WRKDAY:" $WRKDAY + +ANO=`echo ${WRKDAY} | cut -c 1-4` +MES=`echo ${WRKDAY} | cut -c 5-6` +DIA=`echo ${WRKDAY} | cut -c 7-8` +HOR=`echo ${WRKDAY} | cut -c 9-10` + +#DIAFIM=`date +%Y%m%d --date="$ANO$MES$DIA +5 day"` +DIAFIM=`echo ${WRKDAY} 132 | awk -f $path_scr/fwddatan.awk` + +ANOF=`echo ${DIAFIM} | cut -c 1-4` +MESF=`echo ${DIAFIM} | cut -c 5-6` +DIAF=`echo ${DIAFIM} | cut -c 7-8` +HORF=`echo ${DIAFIM} | cut -c 9-10` + +echo $WRFDAT $WPSDIR $WRFDIR $WRKDAY $DIAFIM + +mkdir -p $WPSDIR/$WRKDAY + +echo "SKU_TYPE:" $SKU_TYPE + +SHARED_APP=${SHARED_APP:-/apps} + +# Load environment variables +echo "Load environment variables" +source /data/azurehpc/apps/wrf/env-variables $SKU_TYPE + +echo "SLURM_SUBMIT_DIR:" $SLURM_SUBMIT_DIR + +cd ${WPSDIR}/${WRKDAY} +echo "cd $WPSDIR/$WRKDAY" + +echo "Running link_grib.csh..." +./link_grib.csh ${WRFDAT}/gfs_data/${WRKDAY}/gfs* + +echo "Running ungrib.exe..." +./ungrib.exe >& ungrib.log + +# Capture end time +end_time=$(date +%s) +# Calculate duration +duration=$((end_time - start_time)) +# Format duration to hh:mm:ss +hours=$(printf "%02d" $((duration / 3600))) +minutes=$(printf "%02d" $(( (duration % 3600) / 60 ))) +seconds=$(printf "%02d" $((duration % 60))) +duration="${hours}:${minutes}:${seconds}" + +echo "WPS2 finished." +echo "Execution time: $duration seconds" \ No newline at end of file diff --git a/apps/wrf/automation/run_wps3_openmpi.slurm b/apps/wrf/automation/run_wps3_openmpi.slurm new file mode 100644 index 000000000..f1b177873 --- /dev/null +++ b/apps/wrf/automation/run_wps3_openmpi.slurm @@ -0,0 +1,58 @@ +#!/bin/bash +#SBATCH --job-name=wps3 +#SBATCH --partition=hpc +#SBATCH --ntasks-per-node=16 +#SBATCH --nodes=1 +#SBATCH --exclusive +#SBATCH --output=slurm-wps3-%j.out + +# Capture start time +start_time=$(date +%s) + +SKU_TYPE=${1:-$SKU_TYPE} +SKU_TYPE=${SKU_TYPE:-"hbv3"} + +WRFDAT="/data/wrfdata" +WPSDIR=${WRFDAT}/wpsdir +WRFDIR=${WRFDAT}/wrfdir +WRKDAY=${2:-$WRKDAY} + +echo $WRFDAT $WPSDIR $WRFDIR $WRKDAY + +echo "SKU_TYPE:" $SKU_TYPE + +# Load environment variables +echo "Load environment variables" +source /data/azurehpc/apps/wrf/env-variables $SKU_TYPE + +cd $WPSDIR/$WRKDAY +echo "cd $WPSDIR/$WRKDAY" + +mpi_options="-x LD_LIBRARY_PATH " +if [ -n $LD_PRELOAD ]; then + mpi_options+="-x LD_PRELOAD" +fi +echo "mpi_options:" $mpi_options + +echo "SLURM_JOB_NODELIST:" $SLURM_JOB_NODELIST +echo "SLURM_NTASKS:" $SLURM_NTASKS + +scontrol show hostname $SLURM_JOB_NODELIST > hostfile.txt + +echo "Running geogrid.exe..." +mpirun $mpi_options -n $SLURM_NTASKS --hostfile hostfile.txt --bind-to numa geogrid.exe +echo "Running metgrid.exe..." +mpirun $mpi_options -n $SLURM_NTASKS --hostfile hostfile.txt --bind-to numa metgrid.exe + +# Capture end time +end_time=$(date +%s) +# Calculate duration +duration=$((end_time - start_time)) +# Format duration to hh:mm:ss +hours=$(printf "%02d" $((duration / 3600))) +minutes=$(printf "%02d" $(( (duration % 3600) / 60 ))) +seconds=$(printf "%02d" $((duration % 60))) +duration="${hours}:${minutes}:${seconds}" + +echo "WPS3 finished." +echo "Execution time: $duration seconds" \ No newline at end of file diff --git a/apps/wrf/automation/run_wrf1_openmpi.pbs b/apps/wrf/automation/run_wrf1_openmpi.pbs new file mode 100644 index 000000000..ceacffb80 --- /dev/null +++ b/apps/wrf/automation/run_wrf1_openmpi.pbs @@ -0,0 +1,59 @@ +#!/bin/bash +SKU_TYPE=${1:-$SKU_TYPE} +SKU_TYPE=${SKU_TYPE:-"hbv2"} + +path_scr=/apps/hbv2/scripts +WRFDAT="/data/wrfdata" +WPSDIR=${WRFDAT}/wpsdir +WRFDIR=${WRFDAT}/wrfdir +WRKDAY=${2:-$WRKDAY} + +ANO=`echo ${WRKDAY} | cut -c 1-4` +MES=`echo ${WRKDAY} | cut -c 5-6` +DIA=`echo ${WRKDAY} | cut -c 7-8` +HOR=`echo ${WRKDAY} | cut -c 9-10` + +#DIAFIM=`date +%Y%m%d --date="$ANO$MES$DIA +5 day"` +DIAFIM=`echo ${WRKDAY} 132 | awk -f $path_scr/fwddatan.awk` + +ANOf=`echo ${DIAFIM} | cut -c 1-4` +MESf=`echo ${DIAFIM} | cut -c 5-6` +DIAf=`echo ${DIAFIM} | cut -c 7-8` +HORF=`echo ${DIAFIM} | cut -c 9-10` + +echo $WRFDAT $WPSDIR $WRFDIR + +mkdir -p $WRFDIR/$WRKDAY + +echo "source envs" +source /data/azurehpc/apps/wrf/env-variables + +NPROCS=`cat $PBS_NODEFILE | wc -l` +echo "NPROCS:"$NPROCS + +cd $WRFDIR/$WRKDAY +echo "cd $WRFDIR/$WRKDAY" + +ln -s ${WPSDIR}/${WRKDAY}/met_em* $WRFDIR/$WRKDAY +echo "ln -s ${WPSDIR}/${WRKDAY}/met_em* ." +ln -s ${WRFDAT}/tables/wrf/* $WRFDIR/$WRKDAY +echo "ln -s ${WRFDAT}/tables/wrf/* ." +cp ${WRFDAT}/tables/namelist/namelist.input $WRFDIR/$WRKDAY + +sed -i s/aai/${ANO}/g namelist.input +sed -i s/mmi/${MES}/g namelist.input +sed -i s/ddi/${DIA}/g namelist.input +sed -i s/hhi/${HOR}/g namelist.input + +sed -i s/aaf/${ANOf}/g namelist.input +sed -i s/mmf/${MESf}/g namelist.input +sed -i s/ddf/${DIAf}/g namelist.input +sed -i s/hhf/${HORF}/g namelist.input + +mpi_options="-x LD_LIBRARY_PATH " +if [ -n $LD_PRELOAD ]; then + mpi_options+="-x LD_PRELOAD" +fi +echo "PBS_NODEFILE:" $PBS_NODEFILE + +mpirun $mpi_options -n $NPROCS --hostfile $PBS_NODEFILE --bind-to numa real.exe diff --git a/apps/wrf/automation/run_wrf1_openmpi.slurm b/apps/wrf/automation/run_wrf1_openmpi.slurm new file mode 100644 index 000000000..3ce5bf0fb --- /dev/null +++ b/apps/wrf/automation/run_wrf1_openmpi.slurm @@ -0,0 +1,89 @@ +#!/bin/bash +#SBATCH --job-name=wrf1 +#SBATCH --partition=hpc +#SBATCH --ntasks-per-node=64 +#SBATCH --nodes=1 +#SBATCH --exclusive +#SBATCH --output=slurm-wrf1-%j.out + +# Capture start time +start_time=$(date +%s) + +SKU_TYPE=${1:-$SKU_TYPE} +SKU_TYPE=${SKU_TYPE:-"hbv3"} + +path_scr=/apps/scripts +WRFDAT="/data/wrfdata" +WPSDIR=${WRFDAT}/wpsdir +WRFDIR=${WRFDAT}/wrfdir +WRKDAY=${2:-$WRKDAY} + +ANO=`echo ${WRKDAY} | cut -c 1-4` +MES=`echo ${WRKDAY} | cut -c 5-6` +DIA=`echo ${WRKDAY} | cut -c 7-8` +HOR=`echo ${WRKDAY} | cut -c 9-10` + +#DIAFIM=`date +%Y%m%d --date="$ANO$MES$DIA +5 day"` +DIAFIM=`echo ${WRKDAY} 132 | awk -f $path_scr/fwddatan.awk` + +ANOf=`echo ${DIAFIM} | cut -c 1-4` +MESf=`echo ${DIAFIM} | cut -c 5-6` +DIAf=`echo ${DIAFIM} | cut -c 7-8` +HORF=`echo ${DIAFIM} | cut -c 9-10` + +echo $WRFDAT $WPSDIR $WRFDIR + +mkdir -p $WRFDIR/$WRKDAY + +# Load environment variables +echo "Load environment variables" +source /data/azurehpc/apps/wrf/env-variables $SKU_TYPE + +cd $WRFDIR/$WRKDAY +echo "cd $WRFDIR/$WRKDAY" + +echo "ln -s ${WPSDIR}/${WRKDAY}/met_em* ." +ln -s ${WPSDIR}/${WRKDAY}/met_em* $WRFDIR/$WRKDAY + +echo "ln -s ${WRFDAT}/tables/wrf/* ." +ln -s ${WRFDAT}/tables/wrf/* $WRFDIR/$WRKDAY + +echo "Copying namelist.input" +cp ${WRFDAT}/tables/namelist/namelist.input $WRFDIR/$WRKDAY + +sed -i s/aai/${ANO}/g namelist.input +sed -i s/mmi/${MES}/g namelist.input +sed -i s/ddi/${DIA}/g namelist.input +sed -i s/hhi/${HOR}/g namelist.input + +sed -i s/aaf/${ANOf}/g namelist.input +sed -i s/mmf/${MESf}/g namelist.input +sed -i s/ddf/${DIAf}/g namelist.input +sed -i s/hhf/${HORF}/g namelist.input + +mpi_options="-x LD_LIBRARY_PATH " +if [ -n $LD_PRELOAD ]; then + mpi_options+="-x LD_PRELOAD" +fi +echo "mpi_options:" $mpi_options + +echo "SLURM_JOB_NODELIST:" $SLURM_JOB_NODELIST +echo "SLURM_NTASKS:" $SLURM_NTASKS + +scontrol show hostname $SLURM_JOB_NODELIST > hostfile.txt + +echo "Running real.exe..." +mpirun $mpi_options -n $SLURM_NTASKS --hostfile hostfile.txt --bind-to numa real.exe + +# Capture end time +end_time=$(date +%s) +# Calculate duration +duration=$((end_time - start_time)) +# Format duration to hh:mm:ss +hours=$(printf "%02d" $((duration / 3600))) +minutes=$(printf "%02d" $(( (duration % 3600) / 60 ))) +seconds=$(printf "%02d" $((duration % 60))) +duration="${hours}:${minutes}:${seconds}" + +echo "WRF1 finished." +echo "Execution time: $duration seconds" \ No newline at end of file diff --git a/apps/wrf/automation/run_wrf2_openmpi.pbs b/apps/wrf/automation/run_wrf2_openmpi.pbs new file mode 100644 index 000000000..60560e4d7 --- /dev/null +++ b/apps/wrf/automation/run_wrf2_openmpi.pbs @@ -0,0 +1,35 @@ +#!/bin/bash +SKU_TYPE=${1:-$SKU_TYPE} +SKU_TYPE=${SKU_TYPE:-"hbv2"} + +WRFDAT="/data/wrfdata" +WPSDIR=${WRFDAT}/wpsdir +WRFDIR=${WRFDAT}/wrfdir +WRKDAY=${2:-$WRKDAY} + +ANO=`echo ${WRKDAY} | cut -c 1-4` +MES=`echo ${WRKDAY} | cut -c 5-6` +DIA=`echo ${WRKDAY} | cut -c 7-8` + +echo $WRFDAT $WPSDIR $WRFDIR + +mkdir -p $WRFDIR/$WRKDAY + +echo "SKU_TYPE:" $SKU_TYPE + +echo "source envs" +source /data/azurehpc/apps/wrf/env-variables + +NPROCS=`cat $PBS_NODEFILE | wc -l` +echo "NPROCS:"$NPROCS + +cd $WRFDIR/$WRKDAY +echo "cd $WRFDIR/$WRKDAY" + +mpi_options="-x LD_LIBRARY_PATH " +if [ -n $LD_PRELOAD ]; then + mpi_options+="-x LD_PRELOAD" +fi +echo "PBS_NODEFILE:" $PBS_NODEFILE + +mpirun $mpi_options -n $NPROCS --hostfile $PBS_NODEFILE --bind-to numa wrf.exe diff --git a/apps/wrf/automation/run_wrf2_openmpi.slurm b/apps/wrf/automation/run_wrf2_openmpi.slurm new file mode 100644 index 000000000..cf6bea15f --- /dev/null +++ b/apps/wrf/automation/run_wrf2_openmpi.slurm @@ -0,0 +1,61 @@ +#!/bin/bash +#SBATCH --job-name=wrf2 +#SBATCH --partition=hpc +#SBATCH --ntasks-per-node=96 +#SBATCH --nodes=4 +#SBATCH --exclusive +#SBATCH --output=slurm-wrf2-%j.out + +# Capture start time +start_time=$(date +%s) + +SKU_TYPE=${1:-$SKU_TYPE} +SKU_TYPE=${SKU_TYPE:-"hbv3"} + +WRFDAT="/data/wrfdata" +WPSDIR=${WRFDAT}/wpsdir +WRFDIR=${WRFDAT}/wrfdir +WRKDAY=${2:-$WRKDAY} + +ANO=`echo ${WRKDAY} | cut -c 1-4` +MES=`echo ${WRKDAY} | cut -c 5-6` +DIA=`echo ${WRKDAY} | cut -c 7-8` + +echo $WRFDAT $WPSDIR $WRFDIR + +mkdir -p $WRFDIR/$WRKDAY + +echo "SKU_TYPE:" $SKU_TYPE + +# Load environment variables +echo "Load environment variables" +source /data/azurehpc/apps/wrf/env-variables $SKU_TYPE + +cd $WRFDIR/$WRKDAY +echo "cd $WRFDIR/$WRKDAY" + +mpi_options="-x LD_LIBRARY_PATH " +if [ -n $LD_PRELOAD ]; then + mpi_options+="-x LD_PRELOAD" +fi +echo "mpi_options:" $mpi_options + +echo "SLURM_JOB_NODELIST:" $SLURM_JOB_NODELIST +echo "SLURM_NTASKS:" $SLURM_NTASKS + +scontrol show hostname $SLURM_JOB_NODELIST > hostfile.txt + +mpirun $mpi_options -n $SLURM_NTASKS --hostfile hostfile.txt --bind-to numa wrf.exe + +# Capture end time +end_time=$(date +%s) +# Calculate duration +duration=$((end_time - start_time)) +# Format duration to hh:mm:ss +hours=$(printf "%02d" $((duration / 3600))) +minutes=$(printf "%02d" $(( (duration % 3600) / 60 ))) +seconds=$(printf "%02d" $((duration % 60))) +duration="${hours}:${minutes}:${seconds}" + +echo "WRF2 finished." +echo "Execution time: $duration seconds" \ No newline at end of file diff --git a/apps/wrf/automation/submit-pbs.sh b/apps/wrf/automation/submit-pbs.sh new file mode 100644 index 000000000..d21f39d2e --- /dev/null +++ b/apps/wrf/automation/submit-pbs.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# JOB_wps1 => Executa o download, altera a namelist.wps e executa o ungrib.exe +# JOB_wps2 => Exuta o geogrid.exe e o metgrid.exe +# JOB_wrf1 => Altera a namelist.input e executa o real.exe +# JOB_wrf2 => Executa o wrf.exe +# FILAS => execute1 com 120 ncpus; execute2 com 96 ncpus (otimizada); execute3 com 64 ncpus (otimizada) +# NODES => select 1,2,3 e 4 (maximo de 4 nodes) +# MPIPROCS sempre igual a NCPUS + +path_wrf=/data/wrfdata/wrfdir +path_wps=/data/wrfdata/wpsdir +path_scr=/apps/hbv2/scripts + +data=$1 + +aa=`echo $data | cut -c 1-4` +mm=`echo $data | cut -c 5-6` +dd=`echo $data | cut -c 7-8` + +lastday=`date +%Y%m%d --date="$aa$mm$dd +1 day"` +aaf=`echo $lastday | cut -c 1-4` +mmf=`echo $lastday | cut -c 5-6` +ddf=`echo $lastday | cut -c 7-8` +sfx="_12:00:00" + + +#EST1 => select=(1-4); nodearray=execute1; ncpus=120; mpiprocs=120 +#EST2 => select=(1-4); nodearray=execute2; ncpus=96; mpiprocs=96 +#EST3 => select=(1-4); nodearray=execute3; ncpus=64; mpiprocs=64 + +JOB_wps1=$(qsub -N wps1 -l select=1:nodearray=execute2:ncpus=16:mpiprocs=16,place=scatter:excl -v "SKU_TYPE=hbv2,WRKDAY=$data" $path_scr/run_wps1_openmpi.pbs) +JOB_wps2=$(qsub -W depend=afterany:$JOB_wps1 -N wps2 -l select=1:nodearray=execute2:ncpus=16:mpiprocs=16,place=scatter:excl -v "SKU_TYPE=hbv2,WRKDAY=$data" $path_scr/run_wps2_openmpi.pbs) +JOB_wrf1=$(qsub -W depend=afterany:$JOB_wps2 -N wrf1 -l select=1:nodearray=execute2:ncpus=64:mpiprocs=64,place=scatter:excl -v "SKU_TYPE=hbv2,WRKDAY=$data" $path_scr/run_wrf1_openmpi.pbs) +JOB_wrf2=$(qsub -W depend=afterany:$JOB_wrf1 -N wrf2 -l select=4:nodearray=execute2:ncpus=96:mpiprocs=96,place=scatter:excl -v "SKU_TYPE=hbv2,WRKDAY=$data" $path_scr/run_wrf2_openmpi.pbs) + +exit + +JOB_wps1=$(qsub -N wps1 -l select=1:nodearray=execute3:ncpus=16:mpiprocs=16,place=scatter:excl -v "SKU_TYPE=hbv2,WRKDAY=$data" $path_scr/run_wps1_openmpi.pbs) +JOB_wps2=$(qsub -W depend=afterany:$JOB_wps1 -N wps2 -l select=1:nodearray=execute3:ncpus=16:mpiprocs=16,place=scatter:excl -v "SKU_TYPE=hbv2,WRKDAY=$data" $path_scr/run_wps2_openmpi.pbs) + +exit diff --git a/apps/wrf/automation/submit-slurm.sh b/apps/wrf/automation/submit-slurm.sh new file mode 100644 index 000000000..d938b7863 --- /dev/null +++ b/apps/wrf/automation/submit-slurm.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# JOB_wps1 => Executa o download, altera a namelist.wps +# JOB_wps2 => Executa o ungrib.exe +# JOB_wps3 => Exuta o geogrid.exe e o metgrid.exe +# JOB_wrf1 => Altera a namelist.input e executa o real.exe +# JOB_wrf2 => Executa o wrf.exe + +path_wrf=/data/wrfdata/wrfdir +path_wps=/data/wrfdata/wpsdir +path_scr=/apps/scripts + +data=$1 +echo "data: $data" +mkdir -p $path_wps/$data +mkdir -p $path_wrf/$data + +#sbatch --export="ALL,SKU_TYPE=hbv3,WRKDAY=$data" $path_scr/run_wps2_openmpi.slurm + +# ncpus=16 / 1 node +JOB_wps1=$(sbatch --parsable --export="ALL,SKU_TYPE=hbv3,WRKDAY=$data" $path_scr/run_wps1_openmpi.slurm) +# ncpus=16 / 1 node +JOB_wps2=$(sbatch --parsable --dependency=afterany:$JOB_wps1 --export="ALL,SKU_TYPE=hbv3,WRKDAY=$data" $path_scr/run_wps2_openmpi.slurm) +# ncpus=16 / 1 node +JOB_wps3=$(sbatch --parsable --dependency=afterany:$JOB_wps2 --export="ALL,SKU_TYPE=hbv3,WRKDAY=$data" $path_scr/run_wps3_openmpi.slurm) +# ncpus=64 / 1 node +JOB_wrf1=$(sbatch --parsable --dependency=afterany:$JOB_wps3 --export="ALL,SKU_TYPE=hbv3,WRKDAY=$data" $path_scr/run_wrf1_openmpi.slurm) +# ncpus=96 / 4 nodes +JOB_wrf2=$(sbatch --parsable --dependency=afterany:$JOB_wrf1 --export="ALL,SKU_TYPE=hbv3,WRKDAY=$data" $path_scr/run_wrf2_openmpi.slurm) + +exit diff --git a/apps/wrf/build_wps.sh b/apps/wrf/build_wps.sh index f781c361f..b8cc68ff0 100755 --- a/apps/wrf/build_wps.sh +++ b/apps/wrf/build_wps.sh @@ -27,7 +27,7 @@ function get_version { WRF_VERSION=4.1.5 case $MPI_TYPE in openmpi) - MPI_VER=4.0.5 + MPI_VER=5.0.2 ;; mvapich2) MPI_VER=2.3.5 diff --git a/apps/wrf/build_wrf.sh b/apps/wrf/build_wrf.sh index 59d4c02f3..09180c8fe 100755 --- a/apps/wrf/build_wrf.sh +++ b/apps/wrf/build_wrf.sh @@ -38,7 +38,7 @@ function get_version { MPI_VER=2.7.4 ;; openmpi) - MPI_VER=4.0.5 + MPI_VER=5.0.2 ;; mvapich2) MPI_VER=2.3.5 diff --git a/apps/wrf/cluster-init-projects/import-template.sh b/apps/wrf/cluster-init-projects/import-template.sh new file mode 100644 index 000000000..8f534194b --- /dev/null +++ b/apps/wrf/cluster-init-projects/import-template.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +template_name="Slurm-WRF" +template_file="/anf-vol1/wrf/data/azurehpc/apps/wrf/cluster-init-projects/wrf-proj/templates/slurm-wrf-template-login-node.txt" + +echo "Importing template $template_name from $template_file" + +# Import template +cyclecloud import_template $template_name -f $template_file --force + +echo "Template $template_name imported" diff --git a/apps/wrf/cluster-init-projects/upload-cluster-init-project.sh b/apps/wrf/cluster-init-projects/upload-cluster-init-project.sh new file mode 100644 index 000000000..7a77e454a --- /dev/null +++ b/apps/wrf/cluster-init-projects/upload-cluster-init-project.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +project_folder_name="wrf-proj" +# check locker name: +#cyclecloud locker list +locker_name="GBB HPC-storage" + +echo "Uploading project $project_folder_name to locker $locker_name" + +cd $project_folder_name + +# Upload project +cyclecloud project upload "$locker_name" + +echo "Project $project_folder_name uploaded to locker $locker_name" \ No newline at end of file diff --git a/apps/wrf/cluster-init-projects/wrf-proj/project.ini b/apps/wrf/cluster-init-projects/wrf-proj/project.ini new file mode 100644 index 000000000..bbbe97ac7 --- /dev/null +++ b/apps/wrf/cluster-init-projects/wrf-proj/project.ini @@ -0,0 +1,5 @@ +[project] +name = wrf-proj +type = application +version = 1.0.0 + diff --git a/apps/wrf/cluster-init-projects/wrf-proj/specs/default/cluster-init/files/README.txt b/apps/wrf/cluster-init-projects/wrf-proj/specs/default/cluster-init/files/README.txt new file mode 100644 index 000000000..e110007a4 --- /dev/null +++ b/apps/wrf/cluster-init-projects/wrf-proj/specs/default/cluster-init/files/README.txt @@ -0,0 +1,5 @@ + +Files in this directory are automatically synced to any node using this spec. Content here +can be anything from software packages to config files. Scripts can be used to install +software packages or move files into the appropriate location on the node. + \ No newline at end of file diff --git a/apps/wrf/cluster-init-projects/wrf-proj/specs/default/cluster-init/scripts/001.script1.sh b/apps/wrf/cluster-init-projects/wrf-proj/specs/default/cluster-init/scripts/001.script1.sh new file mode 100644 index 000000000..4c91a85ae --- /dev/null +++ b/apps/wrf/cluster-init-projects/wrf-proj/specs/default/cluster-init/scripts/001.script1.sh @@ -0,0 +1,22 @@ +#! /bin/bash +#set -ex + +# Capture start time +start_time=$(date +%s) + +exec 1>/var/log/001-script-cluster-init.log 2>&1 +echo "### Hello world!" + +# Capture end time +end_time=$(date +%s) +# Calculate duration +duration=$((end_time - start_time)) +# Format duration to hh:mm:ss +hours=$(printf "%02d" $((duration / 3600))) +minutes=$(printf "%02d" $(( (duration % 3600) / 60 ))) +seconds=$(printf "%02d" $((duration % 60))) +duration="${hours}:${minutes}:${seconds}" + +echo "script finished." +echo "Execution time (hh:mm:ss): $duration" + diff --git a/apps/wrf/cluster-init-projects/wrf-proj/specs/default/cluster-init/scripts/README.txt b/apps/wrf/cluster-init-projects/wrf-proj/specs/default/cluster-init/scripts/README.txt new file mode 100644 index 000000000..66c952f98 --- /dev/null +++ b/apps/wrf/cluster-init-projects/wrf-proj/specs/default/cluster-init/scripts/README.txt @@ -0,0 +1,10 @@ + +Files in this directory are executed on the host in alphabetical order. +It is recommended that files are named start with digits to ensure they +are executed in the correct order, example: + - 000_run_me_first.sh + - 001_run_me_second.sh + +Allowable file extensions on Linux: .sh +Allowable file extensions on Windows: .bat, .cmd, .exe + \ No newline at end of file diff --git a/apps/wrf/cluster-init-projects/wrf-proj/specs/default/cluster-init/tests/README.txt b/apps/wrf/cluster-init-projects/wrf-proj/specs/default/cluster-init/tests/README.txt new file mode 100644 index 000000000..a7149f8b1 --- /dev/null +++ b/apps/wrf/cluster-init-projects/wrf-proj/specs/default/cluster-init/tests/README.txt @@ -0,0 +1,5 @@ + +Files in this directory contains tests that will be run at cluster start +when in testing mode. Please see the official documentation for more information +on cluster testing. + \ No newline at end of file diff --git a/apps/wrf/cluster-init-projects/wrf-proj/templates/opbswrf-template-v2.txt b/apps/wrf/cluster-init-projects/wrf-proj/templates/opbswrf-template-v2.txt new file mode 100644 index 000000000..2f02d6c51 --- /dev/null +++ b/apps/wrf/cluster-init-projects/wrf-proj/templates/opbswrf-template-v2.txt @@ -0,0 +1,484 @@ + +################################ +## Cluster Configuration File ## +################################ + +[cluster OpenPBS-WRF] +FormLayout = selectionpanel +IconUrl = https://avatars.githubusercontent.com/u/12666893?s=200&v=4 +Category = Schedulers + +Autoscale = $Autoscale + + [[node defaults]] + UsePublicNetwork = $UsePublicNetwork + Credentials = $Credentials + ImageName = $ImageName + SubnetId = $SubnetId + Region = $Region + KeyPairLocation = ~/.ssh/cyclecloud.pem + Azure.Identities = $ManagedIdentity + + [[[configuration]]] + pbspro.version = $PBSVersion + # For fast spin-up after Deallocate, force an immediate re-converge on boot + cyclecloud.converge_on_boot = true + + # Disable normal NFS exports and mounts + cyclecloud.mounts.sched.disabled = true + cyclecloud.mounts.shared.disabled = true + cyclecloud.exports.sched.disabled = true + cyclecloud.exports.shared.disabled = true + cyclecloud.exports.sched.samba.enabled = false + cyclecloud.exports.shared.samba.enabled = false + cyclecloud.exports.defaults.samba.enabled = false + cshared.server.legacy_links_disabled = true + + # May be used to identify the ID in cluster-init scripts + cluster.identities.default = $ManagedIdentity + + [[[cluster-init cyclecloud/pbspro:default]]] + Optional = false + + + [[[volume boot]]] + Size = ${ifThenElse(BootDiskSize > 0, BootDiskSize, undefined)} + SSD = True + + [[[configuration cyclecloud.mounts.nfs_shared]]] + type = nfs + mountpoint = /shared + export_path = $NFSSharedExportPath + address = $NFSAddress + options = $NFSSharedMountOptions + + [[[configuration cyclecloud.mounts.nfs_sched]]] + type = nfs + mountpoint = /sched + disabled = $NFSSchedDisable + + [[[configuration cyclecloud.mounts.additional_nfs1]]] + disabled = ${AdditionalNAS1 isnt true} + type = nfs + address = $AdditonalNFSAddress1 + mountpoint = $AdditionalNFSMountPoint1 + export_path = $AdditionalNFSExportPath1 + options = $AdditionalNFSMountOptions1 + + + [[[configuration cyclecloud.mounts.additional_nfs2]]] + disabled = ${AdditionalNAS2 isnt true} + type = nfs + address = $AdditonalNFSAddress2 + mountpoint = $AdditionalNFSMountPoint2 + export_path = $AdditionalNFSExportPath2 + options = $AdditionalNFSMountOptions2 + + + [[node server]] + ImageName = $SchedulerImageName + MachineType = $serverMachineType + IsReturnProxy = $ReturnProxy + AdditionalClusterInitSpecs = $serverClusterInitSpecs + + [[[configuration]]] + cyclecloud.mounts.nfs_sched.disabled = true + cyclecloud.mounts.nfs_shared.disabled = ${NFSType != "External"} + pbspro.cron_method = $AzpbsCronMethod + pbspro.queues.workq.ignore = ${Autoscale != true} + pbspro.queues.htcq.ignore = ${Autoscale != true} + + [[[cluster-init cyclecloud/pbspro:server]]] + + [[[network-interface eth0]]] + AssociatePublicIpAddress = $UsePublicNetwork + + [[[input-endpoint ganglia]]] + PrivatePort = 8652 + PublicPort = 8652 + + [[[volume sched]]] + Size = 1024 + SSD = True + Mount = builtinsched + Persistent = False + + [[[volume shared]]] + Size = ${ifThenElse(NFSType == "Builtin", FilesystemSize, 2)} + SSD = True + Mount = builtinshared + Persistent = ${NFSType == "Builtin"} + + [[[configuration cyclecloud.mounts.builtinsched]]] + mountpoint = /sched + fs_type = xfs + + [[[configuration cyclecloud.mounts.builtinshared]]] + disabled = ${NFSType != "Builtin"} + mountpoint = /shared + fs_type = xfs + + [[[configuration cyclecloud.exports.builtinsched]]] + export_path = /sched + options = no_root_squash + samba.enabled = false + type = nfs + + [[[configuration cyclecloud.exports.builtinshared]]] + disabled = ${NFSType != "Builtin"} + export_path = /shared + samba.enabled = false + type = nfs + + + + [[nodearray login]] + InitialCount = $NumberLoginNodes + MachineType = $serverMachineType + + [[[cluster-init cyclecloud/pbspro:login]]] + + [[[configuration]]] + autoscale.enabled = false + + [[nodearray execute]] + MachineType = $ExecuteMachineType + MaxCoreCount = $MaxExecuteCoreCount + + Interruptible = $UseLowPrio + AdditionalClusterInitSpecs = $ExecuteClusterInitSpecs + EnableNodeHealthChecks = $EnableNodeHealthChecks + + + [[[configuration]]] + # setting this completely disables the autoscaler from adding nodes to the cluster + # even manually created nodes. Instead, you can add this to your autoscale.json + # {"pbspro": {"ignore_queues": ["workq", "htcq"]}} + # which is what pbspro.queues.workq.ignore does as well. + # autoscale.enabled = $Autoscale + + [[[cluster-init cyclecloud/pbspro:execute]]] + + [[[network-interface eth0]]] + AssociatePublicIpAddress = $ExecuteNodesPublic + + +[parameters About] +Order = 1 + + [[parameters About OpenPBS]] + + [[[parameter pbspro]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template := "

OpenPBS is a highly configurable open source workload manager. See the OpenPBS project site for an overview.

" + +[parameters Required Settings] +Order = 10 + + [[parameters Virtual Machines ]] + Description = "The cluster, in this case, has two roles: the scheduler server-node with shared filer and the execute hosts. Configure which VM types to use based on the requirements of your application." + Order = 20 + + [[[parameter Region]]] + Label = Region + Description = Deployment Location + ParameterType = Cloud.Region + + [[[parameter serverMachineType]]] + Label = Server VM Type + Description = The VM type for scheduler server and shared filer. + ParameterType = Cloud.MachineType + DefaultValue = Standard_D8as_v4 + + [[[parameter ExecuteMachineType]]] + Label = Execute VM Type + Description = The VM type for execute nodes + ParameterType = Cloud.MachineType + DefaultValue = Standard_F2s_v2 + Config.Multiselect = true + + + [[parameters Auto-Scaling]] + Description = "The cluster can autoscale to the workload, adding execute hosts as jobs are queued. To enable this, check the box below and choose the initial and maximum core counts for the cluster." + Order = 30 + + [[[parameter Autoscale]]] + Label = Autoscale + DefaultValue = true + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Start instances automatically. Shutdown is determined by Keep Alive. + + [[[parameter AzpbsCronMethod]]] + Label = Cron Method + DefaultValue = cron + Config.Plugin = pico.form.Dropdown + Config.Entries := {[Label="Cron"; Value="cron"], [Label="PBS Timer Hook"; Value="pbs_hook"]} + Config.Options = cron,pbs_hook + Description = "The method used to run the azpbs cron job. Cron is the default and will run the azpbs cron job 15 seconds. The pbs_hook method will do the same but use PBS built-in timer." + + [[[parameter MaxExecuteCoreCount]]] + Label = Max Cores + Description = The total number of execute cores to start + DefaultValue = 100 + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 1 + Config.IntegerOnly = true + + [[[parameter UseLowPrio]]] + Label = Low Priority + DefaultValue = false + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Use low priority instances for execute hosts + + [[parameters Networking]] + Order = 40 + + [[[parameter SubnetId]]] + Label = Subnet ID + Description = Subnet Resource Path (ResourceGroup/VirtualNetwork/Subnet) + ParameterType = Azure.Subnet + Required = True + +[parameters Network Attached Storage] +Order = 15 + + + [[parameters Scheduler Mount]] + Order = 5 + [[[parameter About sched]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''

The directory /sched is a network attached mount and exists in all nodes of the cluster. + It's managed by the scheduler node. + To disable the mount of the /sched directory, and to supply your own for a hybrid scenario, select the checkbox below.''' + Order = 6 + + [[[parameter NFSSchedDisable]]] + HideLabel = true + DefaultValue = false + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = External Scheduler + + [[parameters Default NFS Share]] + Order = 10 + [[[parameter About shared]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template := "

The directory /shared is a network attached mount and exists in all nodes of the cluster. Users' home directories reside within this mountpoint with the base homedir /shared/home.

There are two options for providing this mount:
[Builtin]: The scheduler node is an NFS server that provides the mountpoint to the other nodes of the cluster.
[External NFS]: A network attached storage such as Azure Netapp Files, HPC Cache, or another VM running an NFS server, provides the mountpoint.

Note: the cluster must be terminated for this to take effect.

" + Order = 20 + + [[[parameter NFSType]]] + Label = NFS Type + ParameterType = StringList + Config.Label = Type of NFS to use for this cluster + Config.Plugin = pico.form.Dropdown + Config.Entries := {[Label="External NFS"; Value="External"], [Label="Builtin"; Value="Builtin"]} + DefaultValue = Builtin + + [[[parameter NFSDiskWarning]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template := "

Warning: switching an active cluster over to NFS will delete the shared disk.

" + Conditions.Hidden := NFSType != "External" + + [[[parameter NFSAddress]]] + Label = NFS IP Address + Description = The IP address or hostname of the NFS server. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + Config.ParameterType = String + Conditions.Hidden := NFSType != "External" + + [[[parameter NFSSharedExportPath]]] + Label = Shared Export Path + Description = The path exported by the file system + DefaultValue = /shared + Conditions.Hidden := NFSType != "External" + + [[[parameter NFSSharedMountOptions]]] + Label = NFS Mount Options + Description = NFS Client Mount Options + Conditions.Hidden := NFSType != "External" + + [[[parameter FilesystemSize]]] + Label = Size (GB) + Description = The filesystem size (cannot be changed after initial start) + DefaultValue = 100 + + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 10 + Config.MaxValue = 10240 + Config.IntegerOnly = true + Conditions.Excluded := NFSType != "Builtin" + + [[parameters Additional NFS Mount]] + Order = 20 + [[[parameter Additional NFS Mount Readme]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template := "

Mount another NFS endpoint on the cluster nodes.

" + Order = 20 + + [[[parameter AdditionalNAS1]]] + HideLabel = true + DefaultValue = false + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Add NFS mount + + [[[parameter AdditonalNFSAddress1]]] + Label = NFS IP Address + Description = The IP address or hostname of the NFS server. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + Config.ParameterType = String + Conditions.Excluded := AdditionalNAS1 isnt true + + [[[parameter AdditionalNFSMountPoint1]]] + Label = NFS Mount Point + Description = The path at which to mount the Filesystem + DefaultValue = /data + Conditions.Excluded := AdditionalNAS1 isnt true + + [[[parameter AdditionalNFSExportPath1]]] + Label = NFS Export Path + Description = The path exported by the file system + DefaultValue = /anf-vol2/wrf/data + Conditions.Excluded := AdditionalNAS1 isnt true + + [[[parameter AdditionalNFSMountOptions1]]] + Label = NFS Mount Options + Description = NFS Client Mount Options + Conditions.Excluded := AdditionalNAS1 isnt true + + [[[parameter AdditionalNAS2]]] + HideLabel = true + DefaultValue = false + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Add NFS mount + + [[[parameter AdditonalNFSAddress2]]] + Label = NFS IP Address + Description = The IP address or hostname of the NFS server. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + Config.ParameterType = String + Conditions.Excluded := AdditionalNAS2 isnt true + + [[[parameter AdditionalNFSMountPoint2]]] + Label = NFS Mount Point + Description = The path at which to mount the Filesystem + DefaultValue = /apps + Conditions.Excluded := AdditionalNAS2 isnt true + + [[[parameter AdditionalNFSExportPath2]]] + Label = NFS Export Path + Description = The path exported by the file system + DefaultValue = /anf-vol2/wrf/apps + Conditions.Excluded := AdditionalNAS2 isnt true + + [[[parameter AdditionalNFSMountOptions2]]] + Label = NFS Mount Options + Description = NFS Client Mount Options + Conditions.Excluded := AdditionalNAS2 isnt true + + +[parameters Advanced Settings] +Order = 20 + + [[parameters Azure Settings]] + Order = 10 + + [[[parameter Credentials]]] + Description = The credentials for the cloud provider + ParameterType = Cloud.Credentials + + [[[parameter ManagedIdentity]]] + Label = Managed Id + Description = Optionally assign an Azure user assigned managed identity to all nodes to access Azure resources using assigned roles. + ParameterType = Azure.ManagedIdentity + DefaultValue = =undefined + + [[[parameter BootDiskSize]]] + Description = Optional: Size of the OS/boot disk in GB for all nodes in the cluster (leave at 0 to use Image size) + ParameterType = Integer + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 0 + Config.MaxValue = 32,000 + Config.IntegerOnly = true + Config.Increment = 64 + DefaultValue = 0 + + + + [[parameters Software]] + Description = "Specify the scheduling software, and base OS installed on all nodes, and optionally the cluster-init and chef versions from your locker." + Order = 10 + + + [[[parameter SchedulerImageName]]] + Label = Scheduler OS + ParameterType = Cloud.Image + Config.OS = linux + DefaultValue = almalinux8 + Config.Filter := Package in {"cycle.image.centos7", "almalinux8"} + + [[[parameter ImageName]]] + Label = Compute OS + ParameterType = Cloud.Image + Config.OS = linux + DefaultValue = almalinux8 + Config.Filter := Package in {"cycle.image.centos7", "almalinux8"} + + [[[parameter PBSVersion]]] + Label = PBS Version + Config.Plugin = pico.form.Dropdown + Config.Entries := {[Label="OpenPBS v22, el8-only"; Value="22.05.11-0"], [Label="OpenPBS v20, el8-only"; Value="20.0.1-0"], [Label="PBSPro v18, el7-only"; Value="18.1.4-0"]} + DefaultValue = 20.0.1-0 + + [[[parameter serverClusterInitSpecs]]] + Label = Server Cluster-Init + DefaultValue = =undefined + Description = Cluster init specs to apply to the server node + ParameterType = Cloud.ClusterInitSpecs + + [[[parameter ExecuteClusterInitSpecs]]] + Label = Execute Cluster-Init + DefaultValue = =undefined + Description = Cluster init specs to apply to execute nodes + ParameterType = Cloud.ClusterInitSpecs + + [[[parameter NumberLoginNodes]]] + Label = Num Login Nodes + DefaultValue = 0 + Description = Number of optional login nodes to create. + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 0 + Config.MaxValue = 10000 + Config.IntegerOnly = true + + + [[parameters Advanced Networking]] + + [[[parameter ReturnProxy]]] + Label = Return Proxy + DefaultValue = true + ParameterType = Boolean + Config.Label = Use SSH tunnel to connect to CycleCloud (required if direct access is blocked) + + [[[parameter UsePublicNetwork]]] + Label = Public Head Node + DefaultValue = true + ParameterType = Boolean + Config.Label = Access server node from the Internet + + [[[parameter ExecuteNodesPublic]]] + Label = Public Execute + DefaultValue = false + ParameterType = Boolean + Config.Label = Access execute nodes from the Internet + Conditions.Excluded := UsePublicNetwork isnt true + + [[parameters Node Health Checks]] + Description = "Section for configuring Node Health Checks" + Order = 12 + + [[[parameter EnableNodeHealthChecks]]] + Label = Enable NHC tests + DefaultValue = false + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Run Node Health Checks on startup \ No newline at end of file diff --git a/apps/wrf/cluster-init-projects/wrf-proj/templates/opbswrf-template.txt b/apps/wrf/cluster-init-projects/wrf-proj/templates/opbswrf-template.txt new file mode 100644 index 000000000..9ec38a020 --- /dev/null +++ b/apps/wrf/cluster-init-projects/wrf-proj/templates/opbswrf-template.txt @@ -0,0 +1,505 @@ + +################################ +## Cluster Configuration File ## +################################ + +[cluster opbswrf] +FormLayout = selectionpanel +IconUrl = https://avatars.githubusercontent.com/u/12666893?s=200&v=4 +Category = Schedulers + + [[node defaults]] + UsePublicNetwork = $UsePublicNetwork + Credentials = $Credentials + ImageName = $ImageName + SubnetId = $SubnetId + Region = $Region + KeyPairLocation = ~/.ssh/cyclecloud.pem + + [[[configuration]]] + pbspro.version = $PBSVersion + # For fast spin-up after Deallocate, force an immediate re-converge on boot + cyclecloud.converge_on_boot = true + + # Disable normal NFS exports and mounts + cyclecloud.mounts.sched.disabled = true + cyclecloud.mounts.shared.disabled = true + cyclecloud.exports.sched.disabled = true + cyclecloud.exports.shared.disabled = true + cyclecloud.exports.sched.samba.enabled = false + cyclecloud.exports.shared.samba.enabled = false + cyclecloud.exports.defaults.samba.enabled = false + cshared.server.legacy_links_disabled = true + + [[[cluster-init cyclecloud/pbspro:default]]] + Optional = false + + [[[configuration cyclecloud.mounts.nfs_shared]]] + type = nfs + mountpoint = /shared + export_path = $NFSSharedExportPath + address = $NFSAddress + options = $NFSSharedMountOptions + + [[[configuration cyclecloud.mounts.nfs_sched]]] + type = nfs + mountpoint = /sched + + [[[configuration cyclecloud.mounts.additional_nfs1]]] + disabled = ${AdditionalNAS1 isnt true} + type = nfs + address = $AdditonalNFSAddress1 + mountpoint = $AdditionalNFSMountPoint1 + export_path = $AdditionalNFSExportPath1 + options = $AdditionalNFSMountOptions1 + + + [[[configuration cyclecloud.mounts.additional_nfs2]]] + disabled = ${AdditionalNAS2 isnt true} + type = nfs + address = $AdditonalNFSAddress2 + mountpoint = $AdditionalNFSMountPoint2 + export_path = $AdditionalNFSExportPath2 + options = $AdditionalNFSMountOptions2 + + + [[node server]] + ImageName = $SchedulerImageName + MachineType = $serverMachineType + IsReturnProxy = $ReturnProxy + AdditionalClusterInitSpecs = $serverClusterInitSpecs + + [[[configuration]]] + cyclecloud.mounts.nfs_sched.disabled = true + cyclecloud.mounts.nfs_shared.disabled = ${NFSType != "External"} + + [[[cluster-init cyclecloud/pbspro:server]]] + + [[[network-interface eth0]]] + AssociatePublicIpAddress = $UsePublicNetwork + + [[[volume sched]]] + Size = 1024 + SSD = True + Mount = builtinsched + Persistent = False + + [[[volume shared]]] + Size = ${ifThenElse(NFSType == "Builtin", FilesystemSize, 2)} + SSD = True + Mount = builtinshared + Persistent = ${NFSType == "Builtin"} + + [[[configuration cyclecloud.mounts.builtinsched]]] + mountpoint = /sched + fs_type = xfs + + [[[configuration cyclecloud.mounts.builtinshared]]] + disabled = ${NFSType != "Builtin"} + mountpoint = /shared + fs_type = xfs + + [[[configuration cyclecloud.exports.builtinsched]]] + export_path = /sched + options = no_root_squash + samba.enabled = false + type = nfs + + [[[configuration cyclecloud.exports.builtinshared]]] + disabled = ${NFSType != "Builtin"} + export_path = /shared + samba.enabled = false + type = nfs + + + [[nodearray login]] + InitialCount = $NumberLoginNodes + MachineType = $serverMachineType + + [[[cluster-init cyclecloud/pbspro:login]]] + [[[configuration]]] + autoscale.enabled = false + + [[nodearray execute1]] + MachineType = $Execute1MachineType + MaxCoreCount = $MaxExecute1CoreCount + EphemeralOSDisk = true + Interruptible = $UseLowPrio1 + AdditionalClusterInitSpecs = $Execute1ClusterInitSpecs + + + [[[configuration]]] + autoscale.enabled = true + + [[[cluster-init cyclecloud/pbspro:execute]]] + + [[[network-interface eth0]]] + AssociatePublicIpAddress = $Execute1NodesPublic + + + [[nodearray execute2]] + MachineType = $Execute2MachineType + MaxCoreCount = $MaxExecute2CoreCount + EphemeralOSDisk = true + Interruptible = $UseLowPrio2 + AdditionalClusterInitSpecs = $Execute2ClusterInitSpecs + + + [[[configuration]]] + autoscale.enabled = true + + [[[cluster-init cyclecloud/pbspro:execute]]] + + [[[network-interface eth0]]] + AssociatePublicIpAddress = $Execute2NodesPublic + + + [[nodearray execute3]] + MachineType = $Execute3MachineType + MaxCoreCount = $MaxExecute3CoreCount + EphemeralOSDisk = true + Interruptible = $UseLowPrio3 + AdditionalClusterInitSpecs = $Execute3ClusterInitSpecs + + + [[[configuration]]] + autoscale.enabled = true + + [[[cluster-init cyclecloud/pbspro:execute]]] + + [[[network-interface eth0]]] + AssociatePublicIpAddress = $Execute3NodesPublic + + + +[parameters About] +Order = 1 + + [[parameters About OpenPBS]] + + [[[parameter pbspro]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template := "

OpenPBS is a highly configurable open source workload manager. See the OpenPBS project site for an overview.

" + +[parameters Required Settings] +Order = 10 + + [[parameters Virtual Machines ]] + Description = "The cluster, in this case, has two roles: the scheduler server-node with shared filer and the execute hosts. Configure which VM types to use based on the requirements of your application." + Order = 20 + + [[[parameter Region]]] + Label = Region + Description = Deployment Location + ParameterType = Cloud.Region + + [[[parameter serverMachineType]]] + Label = Server VM Type + Description = The VM type for scheduler server and shared filer. + ParameterType = Cloud.MachineType + DefaultValue = Standard_D8as_v4 + + [[[parameter Execute1MachineType]]] + Label = Execute1 VM Type + Description = The VM type for Execute1 nodes + ParameterType = Cloud.MachineType + DefaultValue = Standard_HB120rs_v2 + Config.Multiselect = true + + [[[parameter Execute2MachineType]]] + Label = Execute2 VM Type + Description = The VM type for Execute2 nodes + ParameterType = Cloud.MachineType + DefaultValue = Standard_HB120-32rs_v3 + Config.Multiselect = true + + [[[parameter Execute3MachineType]]] + Label = Execute3 VM Type + Description = The VM type for Execute3 nodes + ParameterType = Cloud.MachineType + DefaultValue = Standard_HB120-64rs_v3 + Config.Multiselect = true + + + [[parameters VM Auto-Scaling]] + Description = "The cluster can autoscale to the workload, adding execute hosts as jobs are queued. To enable this check the box below and choose the initial and maximum core counts for the cluster" + Order = 30 + + [[[parameter MaxExecute1CoreCount]]] + Label = Ex1 Max Cores + Description = The total number of Execute1 cores to start + DefaultValue = 1200 + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 1 + Config.IntegerOnly = true + + [[[parameter MaxExecute2CoreCount]]] + Label = Ex2 Max Cores + Description = The total number of Execute2 cores to start + DefaultValue = 1200 + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 1 + Config.IntegerOnly = true + + [[[parameter MaxExecute3CoreCount]]] + Label = Ex3 Max Cores + Description = The total number of Execute3 cores to start + DefaultValue = 1200 + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 1 + Config.IntegerOnly = true + + [[[parameter UseLowPrio1]]] + Label = Low Priority + DefaultValue = false + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Use low priority instances for Ex1 hosts + + [[[parameter UseLowPrio2]]] + Label = Low Priority + DefaultValue = false + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Use low priority instances for Ex2 hosts + + [[[parameter UseLowPrio3]]] + Label = Low Priority + DefaultValue = false + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Use low priority instances for Ex3 hosts + + [[parameters Networking]] + Order = 40 + + [[[parameter SubnetId]]] + Label = Subnet ID + Description = Subnet Resource Path (ResourceGroup/VirtualNetwork/Subnet) + ParameterType = Azure.Subnet + Required = True + +[parameters Network Attached Storage] +Order = 15 + + [[parameters Default NFS Share]] + Order = 10 + [[[parameter About shared]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template := "

The directory /shared is a network attached mount and exists in all nodes of the cluster. Users' home directories reside within this mountpoint with the base homedir /shared/home.

There are two options for providing this mount:
[Builtin]: The scheduler node is an NFS server that provides the mountpoint to the other nodes of the cluster.
[External NFS]: A network attached storage such as Azure Netapp Files, HPC Cache, or another VM running an NFS server, provides the mountpoint.

" + Order = 20 + + [[[parameter NFSType]]] + Label = NFS Type + ParameterType = StringList + Config.Label = Type of NFS to use for this cluster + Config.Plugin = pico.form.Dropdown + Config.Entries := {[Label="External NFS"; Value="External"], [Label="Builtin"; Value="Builtin"]} + DefaultValue = Builtin + + [[[parameter NFSAddress]]] + Label = NFS IP Address + Description = The IP address or hostname of the NFS server. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + Config.ParameterType = String + Conditions.Hidden := NFSType != "External" + + [[[parameter NFSSharedExportPath]]] + Label = Shared Export Path + Description = The path exported by the file system + DefaultValue = /shared + Conditions.Hidden := NFSType != "External" + + [[[parameter NFSSharedMountOptions]]] + Label = NFS Mount Options + Description = NFS Client Mount Options + Conditions.Hidden := NFSType != "External" + + [[[parameter FilesystemSize]]] + Label = Size (GB) + Description = The filesystem size + DefaultValue = 100 + + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 10 + Config.MaxValue = 10240 + Config.IntegerOnly = true + Conditions.Excluded := NFSType != "Builtin" + + [[parameters Additional NFS Mount]] + Order = 20 + [[[parameter Additional NFS Mount Readme]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template := "

Mount another NFS endpoint on the cluster nodes

" + Order = 20 + + [[[parameter AdditionalNAS1]]] + HideLabel = true + DefaultValue = false + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Add additional NFS mount + + [[[parameter AdditonalNFSAddress1]]] + Label = NFS IP Address + Description = The IP address or hostname of the NFS server. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + Config.ParameterType = String + DefaultValue = "10.4.4.4" + Conditions.Excluded := AdditionalNAS1 isnt true + + [[[parameter AdditionalNFSMountPoint1]]] + Label = NFS Mount Point + Description = The path at which to mount the Filesystem + DefaultValue = /data + Conditions.Excluded := AdditionalNAS1 isnt true + + [[[parameter AdditionalNFSExportPath1]]] + Label = NFS Export Path + Description = The path exported by the file system + DefaultValue = /mnt/exports/data + Conditions.Excluded := AdditionalNAS1 isnt true + + [[[parameter AdditionalNFSMountOptions1]]] + Label = NFS Mount Options + Description = NFS Client Mount Options + Conditions.Excluded := AdditionalNAS1 isnt true + + [[[parameter AdditionalNAS2]]] + HideLabel = true + DefaultValue = false + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Add additional NFS mount + + [[[parameter AdditonalNFSAddress2]]] + Label = NFS IP Address + Description = The IP address or hostname of the NFS server. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + Config.ParameterType = String + DefaultValue = "10.4.4.4" + Conditions.Excluded := AdditionalNAS2 isnt true + + [[[parameter AdditionalNFSMountPoint2]]] + Label = NFS Mount Point + Description = The path at which to mount the Filesystem + DefaultValue = /apps + Conditions.Excluded := AdditionalNAS2 isnt true + + [[[parameter AdditionalNFSExportPath2]]] + Label = NFS Export Path + Description = The path exported by the file system + DefaultValue = /mnt/exports/apps + Conditions.Excluded := AdditionalNAS2 isnt true + + [[[parameter AdditionalNFSMountOptions2]]] + Label = NFS Mount Options + Description = NFS Client Mount Options + Conditions.Excluded := AdditionalNAS2 isnt true + + +[parameters Advanced Settings] +Order = 20 + + [[parameters Azure Settings]] + Order = 10 + + [[[parameter Credentials]]] + Description = The credentials for the cloud provider + ParameterType = Cloud.Credentials + + [[parameters PBSPro Settings ]] + Description = "Section for configuring OpenPBS" + Order = 5 + + + + [[parameters Software]] + Description = "Specify the scheduling software, and base OS installed on all nodes, and optionally the cluster-init and chef versions from your Locker." + Order = 10 + + + [[[parameter SchedulerImageName]]] + Label = Scheduler OS + ParameterType = Cloud.Image + Config.OS = linux + DefaultValue = cycle.image.centos7 + Config.Filter := Package in {"cycle.image.centos7", "cycle.image.centos8"} + + [[[parameter ImageName]]] + Label = Compute OS + ParameterType = Cloud.Image + Config.OS = linux + DefaultValue = cycle.image.centos7 + Config.Filter := Package in {"cycle.image.centos7", "cycle.image.centos8"} + + [[[parameter PBSVersion]]] + Label = PBS Version + Config.Plugin = pico.form.Dropdown + Config.Entries := {[Label="OpenPBS v20, el8-only"; Value="20.0.1-0"], [Label="PBSPro v18, el7-only"; Value="18.1.4-0"]} + DefaultValue = 18.1.4-0 + + [[[parameter serverClusterInitSpecs]]] + Label = Server Cluster-Init + DefaultValue = =undefined + Description = Cluster init specs to apply to the server node + ParameterType = Cloud.ClusterInitSpecs + + [[[parameter Execute1ClusterInitSpecs]]] + Label = Execute1 Cluster-Init + DefaultValue = =undefined + Description = Cluster init specs to apply to execute nodes + ParameterType = Cloud.ClusterInitSpecs + + [[[parameter Execute2ClusterInitSpecs]]] + Label = Execute2 Cluster-Init + DefaultValue = =undefined + Description = Cluster init specs to apply to execute nodes + ParameterType = Cloud.ClusterInitSpecs + + [[[parameter Execute3ClusterInitSpecs]]] + Label = Execute3 Cluster-Init + DefaultValue = =undefined + Description = Cluster init specs to apply to execute 3 nodes + ParameterType = Cloud.ClusterInitSpecs + + [[[parameter NumberLoginNodes]]] + Label = Num Login Nodes + DefaultValue = 0 + Description = Number of optional login nodes to create. + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 0 + Config.MaxValue = 10000 + Config.IntegerOnly = true + + + [[parameters Advanced Networking]] + Description = Advanced networking settings + + [[[parameter ReturnProxy]]] + Label = Return Proxy + DefaultValue = false + ParameterType = Boolean + Config.Label = Use SSH tunnel to connect to CycleCloud (required if direct access is blocked) + + [[[parameter UsePublicNetwork]]] + Label = Public Head Node + DefaultValue = false + ParameterType = Boolean + Config.Label = Access server node from the Internet + + [[[parameter Execute1NodesPublic]]] + Label = Public Execute1 + DefaultValue = false + ParameterType = Boolean + Config.Label = Access execute nodes from the Internet + Conditions.Excluded := UsePublicNetwork isnt true + + [[[parameter Execute2NodesPublic]]] + Label = Public Execute2 + DefaultValue = false + ParameterType = Boolean + Config.Label = Access execute nodes from the Internet + Conditions.Excluded := UsePublicNetwork isnt true + + [[[parameter Execute3NodesPublic]]] + Label = Public Execute3 + DefaultValue = false + ParameterType = Boolean + Config.Label = Access execute nodes from the Internet + Conditions.Excluded := UsePublicNetwork isnt true \ No newline at end of file diff --git a/apps/wrf/cluster-init-projects/wrf-proj/templates/slurm-wrf-template-login-node.txt b/apps/wrf/cluster-init-projects/wrf-proj/templates/slurm-wrf-template-login-node.txt new file mode 100644 index 000000000..d7444a7ae --- /dev/null +++ b/apps/wrf/cluster-init-projects/wrf-proj/templates/slurm-wrf-template-login-node.txt @@ -0,0 +1,962 @@ + +################################ +## Cluster Configuration File ## +################################ + + +[cluster Slurm-WRF] +FormLayout = selectionpanel +IconUrl = https://avatars.githubusercontent.com/u/12666893?s=200&v=4 +Category = Schedulers + +Autoscale = $Autoscale + + [[node defaults]] + UsePublicNetwork = $UsePublicNetwork + Credentials = $Credentials + SubnetId = $SubnetId + Region = $Region + KeyPairLocation = ~/.ssh/cyclecloud.pem + Azure.Identities = $ManagedIdentity + + # Slurm autoscaling supports both Terminate and Deallocate shutdown policies + ShutdownPolicy = $configuration_slurm_shutdown_policy + + # Lustre mounts require termination notifications to unmount + EnableTerminateNotification = ${NFSType == "lustre" || NFSSchedType == "lustre" || AdditionalNFSType1 == "lustre" || AdditionalNFSType2 == "lustre" || EnableTerminateNotification} + TerminateNotificationTimeout = 10m + + [[[configuration]]] + + slurm.install_pkg = azure-slurm-install-pkg-3.0.7.tar.gz + slurm.autoscale_pkg = azure-slurm-pkg-3.0.7.tar.gz + + slurm.version = $configuration_slurm_version + slurm.accounting.enabled = $configuration_slurm_accounting_enabled + slurm.accounting.url = $configuration_slurm_accounting_url + slurm.accounting.user = $configuration_slurm_accounting_user + slurm.accounting.password = $configuration_slurm_accounting_password + slurm.accounting.certificate_url = $configuration_slurm_accounting_certificate_url + slurm.accounting.storageloc = $configuration_slurm_accounting_storageloc + slurm.additional.config = $additional_slurm_config + slurm.ha_enabled = $configuration_slurm_ha_enabled + slurm.launch_parameters = $configuration_slurm_launch_parameters + slurm.disable_pmc = $configuration_slurm_disable_pmc + + # Disable ip-XXXXXXXX hostname generation + cyclecloud.hosts.standalone_dns.enabled = ${NodeNameIsHostname==false} + cyclecloud.hosts.simple_vpc_dns.enabled = ${NodeNameIsHostname==false} + + # For fast spin-up after Deallocate, force an immediate re-converge on boot + cyclecloud.converge_on_boot = true + + # Disable normal NFS exports and mounts + cyclecloud.mounts.sched.disabled = true + cyclecloud.mounts.shared.disabled = true + cyclecloud.exports.sched.disabled = true + cyclecloud.exports.shared.disabled = true + cyclecloud.exports.sched.samba.enabled = false + cyclecloud.exports.shared.samba.enabled = false + cyclecloud.exports.defaults.samba.enabled = false + cshared.server.legacy_links_disabled = true + + # May be used to identify the ID in cluster-init scripts + cluster.identities.default = $ManagedIdentity + + [[[cluster-init cyclecloud/slurm:default:3.0.7]]] + [[[cluster-init wrf-proj:default:1.0.0]]] + + Optional = true + + [[[volume boot]]] + Size = ${ifThenElse(BootDiskSize > 0, BootDiskSize, undefined)} + SSD = True + + [[[configuration cyclecloud.mounts.nfs_shared]]] + type = $NFSType + mountpoint = /shared + export_path = ${ifThenElse(NFSType == "lustre", strcat("tcp:/lustrefs", NFSSharedExportPath), NFSSharedExportPath)} + address = $NFSAddress + options = $NFSSharedMountOptions + + [[[configuration cyclecloud.mounts.nfs_sched]]] + type = $NFSSchedType + mountpoint = /sched + export_path = ${ifThenElse(NFSSchedType == "lustre", strcat("tcp:/lustrefs", NFSSchedExportPath), NFSSchedExportPath)} + address = $NFSSchedAddress + options = $NFSSchedMountOptions + + [[[configuration cyclecloud.mounts.additional_nfs1]]] + disabled = ${AdditionalNFS1 isnt true} + type = $AdditionalNFSType1 + address = $AdditionalNFSAddress1 + mountpoint = $AdditionalNFSMountPoint1 + export_path = ${ifThenElse(AdditionalNFSType1 == "lustre", strcat("tcp:/lustrefs", AdditionalNFSExportPath1), AdditionalNFSExportPath1)} + options = $AdditionalNFSMountOptions1 + + [[[configuration cyclecloud.mounts.additional_nfs2]]] + disabled = ${AdditionalNFS2 isnt true} + type = $AdditionalNFSType2 + address = $AdditionalNFSAddress2 + mountpoint = $AdditionalNFSMountPoint2 + export_path = ${ifThenElse(AdditionalNFSType2 == "lustre", strcat("tcp:/lustrefs", AdditionalNFSExportPath2), AdditionalNFSExportPath2)} + options = $AdditionalNFSMountOptions2 + + [[node scheduler]] + MachineType = $SchedulerMachineType + ImageName = $SchedulerImageName + IsReturnProxy = $ReturnProxy + AdditionalClusterInitSpecs = $SchedulerClusterInitSpecs + ComputerName = ${toLower(regexps("([^a-zA-Z0-9-])", ifThenElse(SchedulerHostName=="Cluster Prefix", StrJoin("-", ClusterName, "scheduler"), ifThenElse(Size(Trim(SchedulerHostName)) == 0 || SchedulerHostName == "Generated", undefined, SchedulerHostName)), "-"))} + # indented version, for clarity. + # ${toLower( + # regexps("([^a-zA-Z0-9-])", + # ifThenElse(SchedulerHostName=="Cluster Prefix", + # StrJoin("-", ClusterName, "scheduler"), + # ifThenElse(Size(Trim(SchedulerHostName)) == 0 || SchedulerHostName == "Generated", + # undefined, + # SchedulerHostName)), + # "-"))} + Zone = ${ifThenElse(configuration_slurm_ha_enabled, SchedulerZone, undefined)} + + [[[configuration]]] + # Disable NFS mount of built-in /sched since it is a local volume mount: cyclecloud.mounts.builtinsched + cyclecloud.mounts.nfs_sched.disabled = ${UseBuiltinSched && !configuration_slurm_ha_enabled} + cyclecloud.mounts.nfs_shared.disabled = ${UseBuiltinShared && !configuration_slurm_ha_enabled} + slurm.secondary_scheduler_name = ${ifThenElse(configuration_slurm_ha_enabled, "scheduler-ha-1", undefined)} + + + [[[cluster-init cyclecloud/slurm:scheduler:3.0.7]]] + + [[[network-interface eth0]]] + AssociatePublicIpAddress = $UsePublicNetwork + + [[[volume sched]]] + Size = $SchedFilesystemSize + SSD = True + Mount = builtinsched + Persistent = False + Disabled = ${!UseBuiltinSched || configuration_slurm_ha_enabled} + + [[[volume shared]]] + Size = $FilesystemSize + SSD = True + Mount = builtinshared + Persistent = True + Disabled = ${!UseBuiltinShared || configuration_slurm_ha_enabled} + + [[[configuration cyclecloud.mounts.builtinsched]]] + disabled = ${!UseBuiltinSched || configuration_slurm_ha_enabled} + mountpoint = /sched + fs_type = xfs + + [[[configuration cyclecloud.mounts.builtinshared]]] + disabled = ${!UseBuiltinShared || configuration_slurm_ha_enabled} + mountpoint = /shared + fs_type = xfs + + [[[configuration cyclecloud.exports.builtinsched]]] + disabled = ${!UseBuiltinSched || configuration_slurm_ha_enabled} + export_path = /sched + options = no_root_squash + samba.enabled = false + type = nfs + + [[[configuration cyclecloud.exports.builtinshared]]] + disabled = ${!UseBuiltinShared || configuration_slurm_ha_enabled} + export_path = /shared + samba.enabled = false + type = nfs + + [[nodearray scheduler-ha]] + Extends = scheduler + IsReturnProxy = false + InitialCount = $configuration_slurm_ha_enabled + Zone = $SchedulerHAZone + [[[configuration]]] + autoscale.enabled = false + slurm.node_prefix = ${ifThenElse(NodeNamePrefix=="Cluster Prefix", StrJoin("-", ClusterName, ""), NodeNamePrefix)} + slurm.use_nodename_as_hostname = $NodeNameIsHostname + slurm.is_primary_scheduler = false + + [[node login]] + InitialCount = $NumberLoginNodes + MachineType = $loginMachineType + ImageName = $SchedulerImageName + SubnetId = "/subscriptions/2d33a7c2-811d-403b-b70d-49d48f690c7c/resourceGroups/mg-rg-hpc/providers/Microsoft.Network/virtualNetworks/mg-vnet-hpc-eastus/subnets/user-subnet" + #itv + #SubnetId = "/subscriptions/09addf29-d567-4494-9d05-db2491ceff53/resourceGroups/azusdsnetrg/providers/Microsoft.Network/virtualNetworks/azusdshpcvnet/subnets/azusdshpcsnet" + + [[[network-interface vm-login-node-cluster2-nic2]]] + #NetworkInterfaceId = "" + PrivateIp = "10.4.2.251" + #itv + #PrivateIp = "10.20.3.61" + + [[[cluster-init cyclecloud/slurm:login:3.0.7]]] + [[[configuration]]] + autoscale.enabled = false + slurm.node_prefix = ${ifThenElse(NodeNamePrefix=="Cluster Prefix", StrJoin("-", ClusterName, ""), NodeNamePrefix)} + slurm.use_nodename_as_hostname = $NodeNameIsHostname + + [[node nodearraybase]] + Abstract = true + [[[configuration]]] + slurm.autoscale = true + + slurm.node_prefix = ${ifThenElse(NodeNamePrefix=="Cluster Prefix", StrJoin("-", ClusterName, ""), NodeNamePrefix)} + slurm.use_nodename_as_hostname = $NodeNameIsHostname + + [[[cluster-init cyclecloud/slurm:execute:3.0.7]]] + + [[[network-interface eth0]]] + AssociatePublicIpAddress = $ExecuteNodesPublic + + [[nodearray hpc]] + Extends = nodearraybase + MachineType = $HPCMachineType + ImageName = $HPCImageName + MaxCoreCount = $MaxHPCExecuteCoreCount + Azure.MaxScalesetSize = $HPCMaxScalesetSize + AdditionalClusterInitSpecs = $HPCClusterInitSpecs + EnableNodeHealthChecks = $EnableNodeHealthChecks + + + [[[configuration]]] + slurm.default_partition = true + slurm.hpc = true + slurm.partition = hpc + + + [[nodearray htc]] + Extends = nodearraybase + MachineType = $HTCMachineType + ImageName = $HTCImageName + MaxCoreCount = $MaxHTCExecuteCoreCount + + Interruptible = $HTCUseLowPrio + MaxPrice = $HTCSpotMaxPrice + AdditionalClusterInitSpecs = $HTCClusterInitSpecs + + [[[configuration]]] + slurm.hpc = false + slurm.partition = htc + # set pcpu = false for all hyperthreaded VMs + slurm.use_pcpu = false + + [[nodearray dynamic]] + Extends = nodearraybase + MachineType = $DynamicMachineType + ImageName = $DynamicImageName + MaxCoreCount = $MaxDynamicExecuteCoreCount + + Interruptible = $DynamicUseLowPrio + MaxPrice = $DynamicSpotMaxPrice + AdditionalClusterInitSpecs = $DynamicClusterInitSpecs + [[[configuration]]] + slurm.hpc = false + slurm.dynamic_config := "-Z --conf \"Feature=dyn\"" + # set pcpu = false for all hyperthreaded VMs + slurm.use_pcpu = false + +[parameters About] +Order = 1 + + [[parameters About Slurm]] + + [[[parameter slurm]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''
Slurm icon

Follow the instructions in theREADMEfor details on instructions on extending and configuring the Project for your environment.


Slurm is the most widely used workload manager in HPC, as the scheduler of choice for six of the top ten systems in the TOP500 and with market penetration of more than 70%. Slurm is an advanced, open-source scheduler designed to satisfy the demanding needs of high-performance computing (HPC), high-throughput computing (HTC), and artificial intelligence (AI).

Commercial Support provided by SchedMD

Get more from your HPC investment! SchedMD, the company behind Slurm development, can answer your Slurm questions and explain our options for consultation, training, support, and migration.

Contact SchedMD

View more details about Slurm?

Slurm at a glance

Slurm provides massive scalability and can easily manage performance requirements for small cluster, large cluster, and supercomputer needs. Slurm outperforms competitive schedulers with compute rates at:

  • 100K+ nodes/GPU
  • 17M+ jobs per day
  • 120M+ jobs per week

Slurm’s plug-in based architecture enables optimization and control in scheduling operations to meet organizational priorities. With first class resource management for GPUs, Slurm allows users to request GPU resources alongside CPUs. This flexibility ensures that jobs are executed quickly and efficiently, while maximizing resource utilization.


Other Slurm features include:

  • NVIDIA and AMD GPU support for AI, LLM, and ML environments
  • Advanced scheduling policies
  • Unique HPC, HTC, AI/ML workload expertise
  • Cloud bursting capabilities
  • Power saving capabilities, accounting, and reporting
  • Provided REST API daemon
  • Native support of containers
  • Tailored Slurm consulting and training available through SchedMD
''' + +[parameters Required Settings] +Order = 10 + + + [[parameters Virtual Machines ]] + Description = "The cluster, in this case, has two roles: the scheduler node with shared filer and the execute hosts. Configure which VM types to use based on the requirements of your application." + Order = 20 + + [[[parameter Region]]] + Label = Region + Description = Deployment Location + ParameterType = Cloud.Region + + [[[parameter SchedulerMachineType]]] + Label = Scheduler VM Type + Description = The VM type for scheduler node + ParameterType = Cloud.MachineType + DefaultValue = Standard_FX4mds + #itv + #DefaultValue = Standard_D4ads_v5 + + [[[parameter loginMachineType]]] + Label = Login node VM Type + Description = The VM type for login nodes. + ParameterType = Cloud.MachineType + DefaultValue = Standard_FX4mds + #itv + #DefaultValue = Standard_D4ads_v5 + + [[[parameter HPCMachineType]]] + Label = HPC VM Type + Description = The VM type for HPC execute nodes + ParameterType = Cloud.MachineType + DefaultValue = Standard_HB120rs_v3 + #itv + #DefaultValue = Standard_HB120rs_v3 + + [[[parameter HTCMachineType]]] + Label = HTC VM Type + Description = The VM type for HTC execute nodes + ParameterType = Cloud.MachineType + DefaultValue = Standard_HB176rs_v4 + #itv + #DefaultValue = Standard_HB176rs_v4 + + [[[parameter DynamicMachineType]]] + Label = Dyn VM Type + Description = The VM type for Dynamic execute nodes + ParameterType = Cloud.MachineType + DefaultValue = Standard_HB120rs_v2 + #itv + #DefaultValue = Standard_HB120rs_v2 + Config.MultiSelect = true + + + [[parameters Auto-Scaling]] + Description = "The cluster can autoscale to the workload, adding execute hosts as jobs are queued. To enable this check the box below and choose the initial and maximum core counts for the cluster." + Order = 30 + + [[[parameter Autoscale]]] + Label = Autoscale + DefaultValue = true + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Start and stop execute instances automatically + + [[[parameter MaxHPCExecuteCoreCount]]] + Label = Max HPC Cores + Description = The total number of HPC execute cores to start + #itv + DefaultValue = 480 + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 1 + Config.IntegerOnly = true + + [[[parameter MaxHTCExecuteCoreCount]]] + Label = Max HTC Cores + Description = The total number of HTC execute cores to start + #itv + DefaultValue = 704 + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 1 + Config.IntegerOnly = true + + [[[parameter MaxDynamicExecuteCoreCount]]] + Label = Max Dyn Cores + Description = The total number of Dynamic execute cores to start + #itv + DefaultValue = 480 + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 1 + Config.IntegerOnly = true + + [[[parameter HPCMaxScalesetSize]]] + Label = Max VMs per VMSS + Description = The maximum number of VMs created per VM Scaleset e.g. switch in Slurm. + DefaultValue = 100 + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 1 + Config.IntegerOnly = true + + + [[[parameter HTCUseLowPrio]]] + Label = HTC Spot + DefaultValue = false + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Use Spot VMs for HTC execute hosts + + [[[parameter HTCSpotMaxPrice]]] + Label = Max Price HTC + DefaultValue = -1 + Description = Max price for Spot VMs in USD (value of -1 will not evict based on price) + Config.Plugin = pico.form.NumberTextBox + Conditions.Excluded := HTCUseLowPrio isnt true + Config.MinValue = -1 + + [[[parameter DynamicUseLowPrio]]] + Label = DynSpot + DefaultValue = false + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Use Spot VMs for Dynamic execute hosts + + [[[parameter DynamicSpotMaxPrice]]] + Label = Max Price Dyn + DefaultValue = -1 + Description = Max price for Spot VMs in USD (value of -1 will not evict based on price) + Config.Plugin = pico.form.NumberTextBox + Conditions.Excluded := DynamicUseLowPrio isnt true + Config.MinValue = -1 + + [[[parameter NumberLoginNodes]]] + Label = Num Login Nodes + DefaultValue = 0 + Description = Number of optional login nodes to create. + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 0 + Config.MaxValue = 10000 + Config.IntegerOnly = true + + [[parameters Networking]] + Order = 40 + + [[[parameter SubnetId]]] + Label = Subnet ID + Description = Subnet Resource Path (ResourceGroup/VirtualNetwork/Subnet) + ParameterType = Azure.Subnet + Required = True + DefaultValue = "mg-rg-hpc/mg-vnet-hpc-eastus/compute-subnet" + #itv + #DefaultValue = "azusdsnetrg/azusdshpcvnet/azusdshpcsnet003" + + [[parameters High Availability]] + Order = 50 + Description = "Slurm can be setup in HA mode - where slurmctld is running on two nodes with failover. Note that checking this box will require an external NFS, so any reference to the 'builtin' NFS will be hidden." + [[[parameter configuration_slurm_ha_enabled]]] + Label = Slurm HA Node + Description = Deploy with an additional HA node + DefaultValue = false + ParameterType = Boolean + + +[parameters Network Attached Storage] +Order = 15 + + [[parameters Shared Storage]] + Order = 10 + + [[[parameter About Shared Storage]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''

The directories /sched and /shared are network attached mounts and exist on all nodes of the cluster.
+
+ Options for providing these mounts:
+ [Builtin]: The scheduler node is an NFS server that provides the mountpoint to the other nodes of the cluster (not supported for HA configurations).
+ [External NFS]: A network attached storage such as Azure Netapp Files, HPC Cache, or another VM running an NFS server provides the mountpoint.
+ [Azure Managed Lustre]: An Azure Managed Lustre deployment provides the mountpoint.
+

+

+ Note: the cluster must be terminated for changes to filesystem mounts to take effect. +

''' + Conditions.Hidden := false + + [[parameters Scheduler Mount]] + Order = 20 + Label = File-system Mount for /sched + + [[[parameter About sched]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''

Slurm's configuration is linked in from the /sched directory. It is managed by the scheduler node

''' + Order = 6 + + [[[parameter About sched part 2]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''

To disable the built-in NFS export of the /sched directory, and to use an external filesystem, select the checkbox below.

''' + Order = 7 + Conditions.Hidden := configuration_slurm_ha_enabled + + [[[parameter UseBuiltinSched]]] + Label = Use Builtin NFS + Description = Use the builtin NFS for /sched + DefaultValue = true + ParameterType = Boolean + Conditions.Hidden := configuration_slurm_ha_enabled + Disabled = configuration_slurm_ha_enabled + + [[[parameter NFSSchedDiskWarning]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template := "

Warning: switching an active cluster over to NFS or Lustre from Builtin will delete the shared disk.

" + Conditions.Hidden := UseBuiltinSched || configuration_slurm_ha_enabled + + [[[parameter NFSSchedType]]] + Label = FS Type + ParameterType = StringList + Config.Label = Type of shared filesystem to use for this cluster + Config.Plugin = pico.form.Dropdown + Config.Entries := {[Label="External NFS"; Value="nfs"], [Label="Azure Managed Lustre"; Value="lustre"]} + DefaultValue = nfs + Conditions.Hidden := UseBuiltinSched && !configuration_slurm_ha_enabled + + [[[parameter NFSSchedAddress]]] + Label = IP Address + Description = The IP address or hostname of the NFS server or Lustre FS. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + Config.ParameterType = String + Conditions.Hidden := UseBuiltinSched && !configuration_slurm_ha_enabled + + [[[parameter NFSSchedExportPath]]] + Label = Export Path + Description = The path exported by the file system + DefaultValue = /sched + Conditions.Hidden := UseBuiltinSched && !configuration_slurm_ha_enabled + + [[[parameter NFSSchedMountOptions]]] + Label = Mount Options + Description = NFS Client Mount Options + Conditions.Hidden := UseBuiltinSched && !configuration_slurm_ha_enabled + + + [[[parameter SchedFilesystemSize]]] + Label = Size (GB) + Description = The filesystem size (cannot be changed after initial start) + DefaultValue = 30 + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 10 + Config.MaxValue = 10240 + Config.IntegerOnly = true + Conditions.Excluded := !UseBuiltinSched || configuration_slurm_ha_enabled + + + + [[parameters Default NFS Share]] + Order = 30 + Label = File-system Mount for /shared + + [[[parameter About shared]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''

Users' home directories reside within the /shared mountpoint with the base homedir /shared/home.

''' + Order = 6 + + [[[parameter About shared part 2]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''

To disable the built-in NFS export of the /shared directory, and to use an external filesystem, select the checkbox below.

''' + Order = 7 + Conditions.Hidden := configuration_slurm_ha_enabled + + [[[parameter UseBuiltinShared]]] + Label = Use Builtin NFS + Description = Use the builtin NFS for /share + DefaultValue = true + ParameterType = Boolean + Conditions.Hidden := configuration_slurm_ha_enabled + Disabled = configuration_slurm_ha_enabled + + [[[parameter NFSDiskWarning]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template := "

Warning: switching an active cluster over to NFS or Lustre from Builtin will delete the shared disk.

" + Conditions.Hidden := UseBuiltinShared || configuration_slurm_ha_enabled + + [[[parameter NFSType]]] + Label = FS Type + ParameterType = StringList + Config.Label = Type of shared filesystem to use for this cluster + Config.Plugin = pico.form.Dropdown + Config.Entries := {[Label="External NFS"; Value="nfs"], [Label="Azure Managed Lustre"; Value="lustre"]} + DefaultValue = nfs + Conditions.Hidden := UseBuiltinShared && !configuration_slurm_ha_enabled + + [[[parameter NFSAddress]]] + Label = IP Address + Description = The IP address or hostname of the NFS server or Lustre FS. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + Config.ParameterType = String + Conditions.Hidden := UseBuiltinShared && !configuration_slurm_ha_enabled + + [[[parameter NFSSharedExportPath]]] + Label = Export Path + Description = The path exported by the file system + DefaultValue = /shared + Conditions.Hidden := UseBuiltinShared && !configuration_slurm_ha_enabled + + [[[parameter NFSSharedMountOptions]]] + Label = Mount Options + Description = NFS Client Mount Options + Conditions.Hidden := UseBuiltinShared && !configuration_slurm_ha_enabled + + + [[[parameter FilesystemSize]]] + Label = Size (GB) + Description = The filesystem size (cannot be changed after initial start) + DefaultValue = 200 + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 10 + Config.MaxValue = 10240 + Config.IntegerOnly = true + Conditions.Excluded := !UseBuiltinShared || configuration_slurm_ha_enabled + + [[parameters Additional NFS Mount]] + Order = 40 + Label = Additional Filesystem Mount + [[[parameter Additional Shared FS Mount Readme]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template := "

Mount another shared filesystem endpoint on the cluster nodes.

" + Order = 20 + + [[[parameter AdditionalNFS1]]] + HideLabel = true + #itv + DefaultValue = true + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Add Shared Filesystem mount + + [[[parameter AdditionalNFSType1]]] + Label = FS Type + ParameterType = StringList + Config.Label = Shared filesystem type of the additional mount + Config.Plugin = pico.form.Dropdown + Config.Entries := {[Label="External NFS"; Value="nfs"], [Label="Azure Managed Lustre"; Value="lustre"]} + DefaultValue = nfs + Conditions.Excluded := AdditionalNFS1 isnt true + + [[[parameter AdditionalNFSAddress1]]] + Label = IP Address + Description = The IP address or hostname of the additional mount. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + Config.ParameterType = String + DefaultValue = "10.4.0.20" + #itv + #DefaultValue = "10.20.3.132" + Conditions.Excluded := AdditionalNFS1 isnt true + + [[[parameter AdditionalNFSMountPoint1]]] + Label = Mount Point + Description = The path at which to mount the Filesystem + #itv + DefaultValue = /data + Conditions.Excluded := AdditionalNFS1 isnt true + + [[[parameter AdditionalNFSExportPath1]]] + Label = Export Path + Description = The path exported by the file system + DefaultValue = /anf-vol1/wrf/data + #itv + #DefaultValue = "/azusdshpcvol/wrf/data" + Conditions.Excluded := AdditionalNFS1 isnt true + + [[[parameter AdditionalNFSMountOptions1]]] + Label = Mount Options + Description = Filesystem Client Mount Options + #itv + DefaultValue = "defaults,rw,hard,rsize=262144,wsize=262144,vers=3,tcp" + Conditions.Excluded := AdditionalNFS1 isnt true + + [[[parameter AdditionalNFS2]]] + HideLabel = true + #itv + DefaultValue = true + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Add Shared Filesystem mount + + [[[parameter AdditionalNFSType2]]] + Label = FS Type + ParameterType = StringList + Config.Label = Shared filesystem type of the additional mount + Config.Plugin = pico.form.Dropdown + Config.Entries := {[Label="External NFS"; Value="nfs"], [Label="Azure Managed Lustre"; Value="lustre"]} + DefaultValue = nfs + Conditions.Excluded := AdditionalNFS2 isnt true + + [[[parameter AdditionalNFSAddress2]]] + Label = IP Address + Description = The IP address or hostname of the additional mount. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + Config.ParameterType = String + DefaultValue = "10.4.0.20" + #itv + #DefaultValue = "10.20.3.132" + Conditions.Excluded := AdditionalNFS2 isnt true + + [[[parameter AdditionalNFSMountPoint2]]] + Label = Mount Point + Description = The path at which to mount the Filesystem + #itv + DefaultValue = /apps + Conditions.Excluded := AdditionalNFS2 isnt true + + [[[parameter AdditionalNFSExportPath2]]] + Label = Export Path + Description = The path exported by the file system + DefaultValue = /anf-vol1/wrf/apps + #itv + #DefaultValue = "/azusdshpcvol/wrf/apps" + Conditions.Excluded := AdditionalNFS2 isnt true + + [[[parameter AdditionalNFSMountOptions2]]] + Label = Mount Options + Description = Filesystem Client Mount Options + #itv + DefaultValue = "defaults,rw,hard,rsize=262144,wsize=262144,vers=3,tcp" + Conditions.Excluded := AdditionalNFS2 isnt true + + +[parameters Advanced Settings] +Order = 20 + + [[parameters Azure Settings]] + Order = 10 + + [[[parameter Credentials]]] + Description = The credentials for the cloud provider + ParameterType = Cloud.Credentials + DefaultValue = "GBB HPC - Default credentials" + #itv + #DefaultValue = "WRF - Default credentials" + + [[[parameter ManagedIdentity]]] + Label = Managed Id + Description = Optionally assign an Azure user assigned managed identity to all nodes to access Azure resources using assigned roles. + ParameterType = Azure.ManagedIdentity + DefaultValue = =undefined + + [[[parameter BootDiskSize]]] + Description = Optional: Size of the OS/boot disk in GB for all nodes in the cluster (leave at 0 to use Image size) + ParameterType = Integer + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 0 + Config.MaxValue = 32,000 + Config.IntegerOnly = true + Config.Increment = 64 + DefaultValue = 0 + + [[parameters Slurm Settings ]] + + Order = 5 + + [[[parameter slurm_version_warning]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template := "
Note: For SLES HPC, we can only install the version supported by SLES HPC's zypper repos. At the time of this release, that is 23.02.7
" + + + [[[parameter configuration_slurm_version]]] + Required = True + Label = Slurm Version + Description = Version of Slurm to install on the cluster + ParameterType = StringList + Config.Plugin = pico.form.Dropdown + Config.FreeForm = true + Config.Entries := {[Value="23.02.7-4"], [Value="23.11.7-1"]} + DefaultValue = 23.11.7-1 + + [[[parameter configuration_slurm_accounting_enabled]]] + Label = Job Accounting + DefaultValue = false + #itv + #DefaultValue = true + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Configure Slurm job accounting + + [[[parameter slurm_database_warning]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Conditions.Excluded := configuration_slurm_accounting_enabled isnt true + Config.Template := "
Note: Checking this box will create persistent databases and tables in SQL DB provided. Deleting this cluster will not automatically delete those databases. User is responsible for periodically purging/archiving their slurm databases to maintain costs.
" + + [[[parameter configuration_slurm_accounting_url]]] + Label = Slurm DBD URL + Description = URL of the database to use for Slurm job accounting + #itv + #DefaultValue = "azusdshpcmysql.mysql.database.azure.com" + Conditions.Excluded := configuration_slurm_accounting_enabled isnt true + + [[[parameter configuration_slurm_accounting_storageloc]]] + Label = Database name + Description = Database name to store slurm accounting records + #itv + #DefaultValue = "wrf" + Conditions.Excluded := configuration_slurm_accounting_enabled isnt true + + [[[parameter configuration_slurm_accounting_user]]] + Label = Slurm DBD User + Description = User for Slurm DBD admin + #itv + #DefaultValue = "dbmysql_ds_usr" + Conditions.Excluded := configuration_slurm_accounting_enabled isnt true + + [[[parameter configuration_slurm_accounting_password]]] + Label = Slurm DBD Password + Description = Password for Slurm DBD admin + ParameterType = Password + #itv + #DefaultValue = "inclua a senha aqui" + Conditions.Excluded := configuration_slurm_accounting_enabled isnt true + + [[[parameter configuration_slurm_accounting_certificate_url]]] + Label = SSL Certificate URL + Description = URL to fetch SSL certificate for authentication to DB. Use AzureCA.pem (embedded) for use with deprecated MariaDB instances. + Conditions.Excluded := configuration_slurm_accounting_enabled isnt true + ParameterType = StringList + Config.Plugin = pico.form.Dropdown + Config.FreeForm = true + Config.Entries := {[Value=""], [Value="AzureCA.pem"]} + #itv + #DefaultValue = "https://dl.cacerts.digicert.com/DigiCertGlobalRootCA.crt.pem" + + [[[parameter configuration_slurm_shutdown_policy]]] + Label = Shutdown Policy + description = By default, autostop will Delete stopped VMS for lowest cost. Optionally, Stop/Deallocate the VMs for faster restart instead. + DefaultValue = Terminate + config.plugin = pico.control.AutoCompleteDropdown + [[[[list Config.Entries]]]] + Name = Terminate + Label = Terminate + [[[[list Config.Entries]]]] + Name = Deallocate + Label = Deallocate + + [[[parameter EnableTerminateNotification]]] + Label = Enable Termination notifications + DefaultValue = False + + [[[parameter additional_slurm_config]]] + Label = Slurm Configuration + Description = Any additional lines to add to slurm.conf + ParameterType = Text + + [[[parameter configuration_slurm_launch_parameters]]] + Label = Launch Parameters + Description = Deploy Slurm with Launch Parameters (comma delimited) + DefaultValue = '' + ParameterType = String + + + + [[parameters Software]] + Description = "Specify the scheduling software, and base OS installed on all nodes, and optionally the cluster-init and chef versions from your locker." + Order = 10 + + [[[parameter NodeNameIsHostname]]] + Label = Name As Hostname + Description = Should the hostname match the nodename for execute nodes? + ParameterType = Boolean + DefaultValue = true + + [[[parameter NodeNamePrefix]]] + Label = Node Prefix + Description = Prefix for generated node names, i.e. "prefix-" generates prefix-nodearray-1. Use 'Cluster Prefix' to get $ClusterName-nodearray-1 + ParameterType = StringList + Config.Plugin = pico.form.Dropdown + Config.FreeForm = true + DefaultValue = "Cluster Prefix" + Config.Entries := {[Value=""], [Value="Cluster Prefix"]} + Conditions.Hidden := NodeNameIsHostname != true + + [[[parameter SchedulerHostName]]] + Label = Scheduler Hostname + Description = Hostname of scheduler. 'Generated' uses the default generated hostname. 'Cluster Prefix' will generate $ClusterName-scheduler. + ParameterType = StringList + Config.Plugin = pico.form.Dropdown + Config.FreeForm = true + DefaultValue = "Cluster Prefix" + Config.Entries := {[Value="Generated"], [Value="Cluster Prefix"]} + Conditions.Hidden := NodeNameIsHostname != true + + [[[parameter SchedulerImageName]]] + Label = Scheduler OS + ParameterType = Cloud.Image + Config.OS = linux + DefaultValue = almalinux8 + Config.Filter := Package in {"cycle.image.centos7", "cycle.image.ubuntu20", "cycle.image.ubuntu22", "cycle.image.sles15-hpc", "almalinux8"} + + [[[parameter HPCImageName]]] + Label = HPC OS + ParameterType = Cloud.Image + Config.OS = linux + DefaultValue = almalinux8 + Config.Filter := Package in {"cycle.image.centos7", "cycle.image.ubuntu20", "cycle.image.ubuntu22", "cycle.image.sles15-hpc", "almalinux8"} + + [[[parameter HTCImageName]]] + Label = HTC OS + ParameterType = Cloud.Image + Config.OS = linux + DefaultValue = almalinux8 + Config.Filter := Package in {"cycle.image.centos7", "cycle.image.ubuntu20", "cycle.image.ubuntu22", "cycle.image.sles15-hpc", "almalinux8"} + + [[[parameter DynamicImageName]]] + Label = Dynamic OS + ParameterType = Cloud.Image + Config.OS = linux + DefaultValue = almalinux8 + Config.Filter := Package in {"cycle.image.centos7", "cycle.image.ubuntu20", "cycle.image.ubuntu22", "cycle.image.sles15-hpc", "almalinux8"} + + [[[parameter SchedulerClusterInitSpecs]]] + Label = Scheduler Cluster-Init + DefaultValue = =undefined + Description = Cluster init specs to apply to the scheduler node + ParameterType = Cloud.ClusterInitSpecs + + [[[parameter HTCClusterInitSpecs]]] + Label = HTC Cluster-Init + DefaultValue = =undefined + Description = Cluster init specs to apply to HTC execute nodes + ParameterType = Cloud.ClusterInitSpecs + + [[[parameter HPCClusterInitSpecs]]] + Label = HPC Cluster-Init + DefaultValue = =undefined + Description = Cluster init specs to apply to HPC execute nodes + ParameterType = Cloud.ClusterInitSpecs + + [[[parameter DynamicClusterInitSpecs]]] + Label = Dyn Cluster-Init + DefaultValue = =undefined + Description = Cluster init specs to apply to Dynamic execute nodes + ParameterType = Cloud.ClusterInitSpecs + + [[[parameter configuration_slurm_disable_pmc]]] + Label = Disable PMC + Description = Disable packages from packages.microsoft.com + ParameterType = Boolean + DefaultValue = false + + + [[parameters Advanced Networking]] + + [[[parameter ReturnProxy]]] + Label = Return Proxy + #itv + DefaultValue = false + ParameterType = Boolean + Config.Label = Use SSH tunnel to connect to CycleCloud (required if direct access is blocked) + + [[[parameter UsePublicNetwork]]] + Label = Public Head Node + #itv + DefaultValue = false + ParameterType = Boolean + Config.Label = Access scheduler node from the Internet + + [[[parameter ExecuteNodesPublic]]] + Label = Public Execute + DefaultValue = false + ParameterType = Boolean + Config.Label = Access execute nodes from the Internet + Conditions.Excluded := UsePublicNetwork isnt true + + [[[parameter SchedulerZone]]] + Label = Scheduler Zone + Description = The availability zone in which to deploy the scheduler node. + DefaultValue = =undefined + Config.Plugin = pico.form.Dropdown + Config.Entries := {[Value=1], [Value=2], [Value=3], [Value=undefined; Label="Any"]} + + [[[parameter SchedulerHAZone]]] + Label = Scheduler HA Zone + Description = The availability zone in which to deploy the scheduler-ha node. + DefaultValue = =undefined + Config.Plugin = pico.form.Dropdown + Config.Entries := {[Value=1], [Value=2], [Value=3], [Value=undefined; Label="Any"]} + Conditions.Hidden := configuration_slurm_ha_enabled isnt true + + [[parameters Node Health Checks]] + Description = "Section for configuring Node Health Checks" + Order = 12 + + [[[parameter EnableNodeHealthChecks]]] + Label = Enable NHC tests + DefaultValue = false + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Run Node Health Checks on startup diff --git a/apps/wrf/cluster-init-projects/wrf-proj/templates/slurm-wrf-template.txt b/apps/wrf/cluster-init-projects/wrf-proj/templates/slurm-wrf-template.txt new file mode 100644 index 000000000..6e89e562c --- /dev/null +++ b/apps/wrf/cluster-init-projects/wrf-proj/templates/slurm-wrf-template.txt @@ -0,0 +1,902 @@ + +################################ +## Cluster Configuration File ## +################################ + + +[cluster Slurm-WRF] +FormLayout = selectionpanel +IconUrl = https://avatars.githubusercontent.com/u/12666893?s=200&v=4 +Category = Schedulers + +Autoscale = $Autoscale + + [[node defaults]] + UsePublicNetwork = $UsePublicNetwork + Credentials = $Credentials + SubnetId = $SubnetId + Region = $Region + KeyPairLocation = ~/.ssh/cyclecloud.pem + Azure.Identities = $ManagedIdentity + + # Slurm autoscaling supports both Terminate and Deallocate shutdown policies + ShutdownPolicy = $configuration_slurm_shutdown_policy + + # Lustre mounts require termination notifications to unmount + EnableTerminateNotification = ${NFSType == "lustre" || NFSSchedType == "lustre" || AdditionalNFSType1 == "lustre" || AdditionalNFSType2 == "lustre" || EnableTerminateNotification} + TerminateNotificationTimeout = 10m + + [[[configuration]]] + + slurm.install_pkg = azure-slurm-install-pkg-3.0.7.tar.gz + slurm.autoscale_pkg = azure-slurm-pkg-3.0.7.tar.gz + + slurm.version = $configuration_slurm_version + slurm.accounting.enabled = $configuration_slurm_accounting_enabled + slurm.accounting.url = $configuration_slurm_accounting_url + slurm.accounting.user = $configuration_slurm_accounting_user + slurm.accounting.password = $configuration_slurm_accounting_password + slurm.accounting.certificate_url = $configuration_slurm_accounting_certificate_url + slurm.accounting.storageloc = $configuration_slurm_accounting_storageloc + slurm.additional.config = $additional_slurm_config + slurm.ha_enabled = $configuration_slurm_ha_enabled + slurm.launch_parameters = $configuration_slurm_launch_parameters + slurm.disable_pmc = $configuration_slurm_disable_pmc + + # Disable ip-XXXXXXXX hostname generation + cyclecloud.hosts.standalone_dns.enabled = ${NodeNameIsHostname==false} + cyclecloud.hosts.simple_vpc_dns.enabled = ${NodeNameIsHostname==false} + + # For fast spin-up after Deallocate, force an immediate re-converge on boot + cyclecloud.converge_on_boot = true + + # Disable normal NFS exports and mounts + cyclecloud.mounts.sched.disabled = true + cyclecloud.mounts.shared.disabled = true + cyclecloud.exports.sched.disabled = true + cyclecloud.exports.shared.disabled = true + cyclecloud.exports.sched.samba.enabled = false + cyclecloud.exports.shared.samba.enabled = false + cyclecloud.exports.defaults.samba.enabled = false + cshared.server.legacy_links_disabled = true + + # May be used to identify the ID in cluster-init scripts + cluster.identities.default = $ManagedIdentity + + [[[cluster-init cyclecloud/slurm:default:3.0.7]]] + + Optional = true + + [[[volume boot]]] + Size = ${ifThenElse(BootDiskSize > 0, BootDiskSize, undefined)} + SSD = True + + [[[configuration cyclecloud.mounts.nfs_shared]]] + type = $NFSType + mountpoint = /shared + export_path = ${ifThenElse(NFSType == "lustre", strcat("tcp:/lustrefs", NFSSharedExportPath), NFSSharedExportPath)} + address = $NFSAddress + options = $NFSSharedMountOptions + + [[[configuration cyclecloud.mounts.nfs_sched]]] + type = $NFSSchedType + mountpoint = /sched + export_path = ${ifThenElse(NFSSchedType == "lustre", strcat("tcp:/lustrefs", NFSSchedExportPath), NFSSchedExportPath)} + address = $NFSSchedAddress + options = $NFSSchedMountOptions + + [[[configuration cyclecloud.mounts.additional_nfs1]]] + disabled = ${AdditionalNFS1 isnt true} + type = $AdditionalNFSType1 + address = $AdditionalNFSAddress1 + mountpoint = $AdditionalNFSMountPoint1 + export_path = ${ifThenElse(AdditionalNFSType1 == "lustre", strcat("tcp:/lustrefs", AdditionalNFSExportPath1), AdditionalNFSExportPath1)} + options = $AdditionalNFSMountOptions1 + + [[[configuration cyclecloud.mounts.additional_nfs2]]] + disabled = ${AdditionalNFS2 isnt true} + type = $AdditionalNFSType2 + address = $AdditionalNFSAddress2 + mountpoint = $AdditionalNFSMountPoint2 + export_path = ${ifThenElse(AdditionalNFSType2 == "lustre", strcat("tcp:/lustrefs", AdditionalNFSExportPath2), AdditionalNFSExportPath2)} + options = $AdditionalNFSMountOptions2 + + [[node scheduler]] + MachineType = $SchedulerMachineType + ImageName = $SchedulerImageName + IsReturnProxy = $ReturnProxy + AdditionalClusterInitSpecs = $SchedulerClusterInitSpecs + ComputerName = ${toLower(regexps("([^a-zA-Z0-9-])", ifThenElse(SchedulerHostName=="Cluster Prefix", StrJoin("-", ClusterName, "scheduler"), ifThenElse(Size(Trim(SchedulerHostName)) == 0 || SchedulerHostName == "Generated", undefined, SchedulerHostName)), "-"))} + # indented version, for clarity. + # ${toLower( + # regexps("([^a-zA-Z0-9-])", + # ifThenElse(SchedulerHostName=="Cluster Prefix", + # StrJoin("-", ClusterName, "scheduler"), + # ifThenElse(Size(Trim(SchedulerHostName)) == 0 || SchedulerHostName == "Generated", + # undefined, + # SchedulerHostName)), + # "-"))} + Zone = ${ifThenElse(configuration_slurm_ha_enabled, SchedulerZone, undefined)} + + [[[configuration]]] + # Disable NFS mount of built-in /sched since it is a local volume mount: cyclecloud.mounts.builtinsched + cyclecloud.mounts.nfs_sched.disabled = ${UseBuiltinSched && !configuration_slurm_ha_enabled} + cyclecloud.mounts.nfs_shared.disabled = ${UseBuiltinShared && !configuration_slurm_ha_enabled} + slurm.secondary_scheduler_name = ${ifThenElse(configuration_slurm_ha_enabled, "scheduler-ha-1", undefined)} + + + [[[cluster-init cyclecloud/slurm:scheduler:3.0.7]]] + + [[[network-interface eth0]]] + AssociatePublicIpAddress = $UsePublicNetwork + + [[[volume sched]]] + Size = $SchedFilesystemSize + SSD = True + Mount = builtinsched + Persistent = False + Disabled = ${!UseBuiltinSched || configuration_slurm_ha_enabled} + + [[[volume shared]]] + Size = $FilesystemSize + SSD = True + Mount = builtinshared + Persistent = True + Disabled = ${!UseBuiltinShared || configuration_slurm_ha_enabled} + + [[[configuration cyclecloud.mounts.builtinsched]]] + disabled = ${!UseBuiltinSched || configuration_slurm_ha_enabled} + mountpoint = /sched + fs_type = xfs + + [[[configuration cyclecloud.mounts.builtinshared]]] + disabled = ${!UseBuiltinShared || configuration_slurm_ha_enabled} + mountpoint = /shared + fs_type = xfs + + [[[configuration cyclecloud.exports.builtinsched]]] + disabled = ${!UseBuiltinSched || configuration_slurm_ha_enabled} + export_path = /sched + options = no_root_squash + samba.enabled = false + type = nfs + + [[[configuration cyclecloud.exports.builtinshared]]] + disabled = ${!UseBuiltinShared || configuration_slurm_ha_enabled} + export_path = /shared + samba.enabled = false + type = nfs + + [[nodearray scheduler-ha]] + Extends = scheduler + IsReturnProxy = false + InitialCount = $configuration_slurm_ha_enabled + Zone = $SchedulerHAZone + [[[configuration]]] + autoscale.enabled = false + slurm.node_prefix = ${ifThenElse(NodeNamePrefix=="Cluster Prefix", StrJoin("-", ClusterName, ""), NodeNamePrefix)} + slurm.use_nodename_as_hostname = $NodeNameIsHostname + slurm.is_primary_scheduler = false + + [[nodearray login]] + InitialCount = $NumberLoginNodes + MachineType = $loginMachineType + ImageName = $SchedulerImageName + + [[[cluster-init cyclecloud/slurm:login:3.0.7]]] + [[[configuration]]] + autoscale.enabled = false + slurm.node_prefix = ${ifThenElse(NodeNamePrefix=="Cluster Prefix", StrJoin("-", ClusterName, ""), NodeNamePrefix)} + slurm.use_nodename_as_hostname = $NodeNameIsHostname + + [[node nodearraybase]] + Abstract = true + [[[configuration]]] + slurm.autoscale = true + + slurm.node_prefix = ${ifThenElse(NodeNamePrefix=="Cluster Prefix", StrJoin("-", ClusterName, ""), NodeNamePrefix)} + slurm.use_nodename_as_hostname = $NodeNameIsHostname + + [[[cluster-init cyclecloud/slurm:execute:3.0.7]]] + + [[[network-interface eth0]]] + AssociatePublicIpAddress = $ExecuteNodesPublic + + [[nodearray hpc]] + Extends = nodearraybase + MachineType = $HPCMachineType + ImageName = $HPCImageName + MaxCoreCount = $MaxHPCExecuteCoreCount + Azure.MaxScalesetSize = $HPCMaxScalesetSize + AdditionalClusterInitSpecs = $HPCClusterInitSpecs + EnableNodeHealthChecks = $EnableNodeHealthChecks + + + [[[configuration]]] + slurm.default_partition = true + slurm.hpc = true + slurm.partition = hpc + + + [[nodearray htc]] + Extends = nodearraybase + MachineType = $HTCMachineType + ImageName = $HTCImageName + MaxCoreCount = $MaxHTCExecuteCoreCount + + Interruptible = $HTCUseLowPrio + MaxPrice = $HTCSpotMaxPrice + AdditionalClusterInitSpecs = $HTCClusterInitSpecs + + [[[configuration]]] + slurm.hpc = false + slurm.partition = htc + # set pcpu = false for all hyperthreaded VMs + slurm.use_pcpu = false + + [[nodearray dynamic]] + Extends = nodearraybase + MachineType = $DynamicMachineType + ImageName = $DynamicImageName + MaxCoreCount = $MaxDynamicExecuteCoreCount + + Interruptible = $DynamicUseLowPrio + MaxPrice = $DynamicSpotMaxPrice + AdditionalClusterInitSpecs = $DynamicClusterInitSpecs + [[[configuration]]] + slurm.hpc = false + slurm.dynamic_config := "-Z --conf \"Feature=dyn\"" + # set pcpu = false for all hyperthreaded VMs + slurm.use_pcpu = false + +[parameters About] +Order = 1 + + [[parameters About Slurm]] + + [[[parameter slurm]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''
Slurm icon

Follow the instructions in theREADMEfor details on instructions on extending and configuring the Project for your environment.


Slurm is the most widely used workload manager in HPC, as the scheduler of choice for six of the top ten systems in the TOP500 and with market penetration of more than 70%. Slurm is an advanced, open-source scheduler designed to satisfy the demanding needs of high-performance computing (HPC), high-throughput computing (HTC), and artificial intelligence (AI).

Commercial Support provided by SchedMD

Get more from your HPC investment! SchedMD, the company behind Slurm development, can answer your Slurm questions and explain our options for consultation, training, support, and migration.

Contact SchedMD

View more details about Slurm?

Slurm at a glance

Slurm provides massive scalability and can easily manage performance requirements for small cluster, large cluster, and supercomputer needs. Slurm outperforms competitive schedulers with compute rates at:

  • 100K+ nodes/GPU
  • 17M+ jobs per day
  • 120M+ jobs per week

Slurm’s plug-in based architecture enables optimization and control in scheduling operations to meet organizational priorities. With first class resource management for GPUs, Slurm allows users to request GPU resources alongside CPUs. This flexibility ensures that jobs are executed quickly and efficiently, while maximizing resource utilization.


Other Slurm features include:

  • NVIDIA and AMD GPU support for AI, LLM, and ML environments
  • Advanced scheduling policies
  • Unique HPC, HTC, AI/ML workload expertise
  • Cloud bursting capabilities
  • Power saving capabilities, accounting, and reporting
  • Provided REST API daemon
  • Native support of containers
  • Tailored Slurm consulting and training available through SchedMD
''' + +[parameters Required Settings] +Order = 10 + + + [[parameters Virtual Machines ]] + Description = "The cluster, in this case, has two roles: the scheduler node with shared filer and the execute hosts. Configure which VM types to use based on the requirements of your application." + Order = 20 + + [[[parameter Region]]] + Label = Region + Description = Deployment Location + ParameterType = Cloud.Region + + [[[parameter SchedulerMachineType]]] + Label = Scheduler VM Type + Description = The VM type for scheduler node + ParameterType = Cloud.MachineType + DefaultValue = Standard_D4ads_v5 + + [[[parameter loginMachineType]]] + Label = Login node VM Type + Description = The VM type for login nodes. + ParameterType = Cloud.MachineType + DefaultValue = Standard_D8as_v4 + + [[[parameter HPCMachineType]]] + Label = HPC VM Type + Description = The VM type for HPC execute nodes + ParameterType = Cloud.MachineType + DefaultValue = Standard_F2s_v2 + + [[[parameter HTCMachineType]]] + Label = HTC VM Type + Description = The VM type for HTC execute nodes + ParameterType = Cloud.MachineType + DefaultValue = Standard_F2s_v2 + + [[[parameter DynamicMachineType]]] + Label = Dyn VM Type + Description = The VM type for Dynamic execute nodes + ParameterType = Cloud.MachineType + DefaultValue = Standard_F2s_v2 + Config.MultiSelect = true + + + [[parameters Auto-Scaling]] + Description = "The cluster can autoscale to the workload, adding execute hosts as jobs are queued. To enable this check the box below and choose the initial and maximum core counts for the cluster." + Order = 30 + + [[[parameter Autoscale]]] + Label = Autoscale + DefaultValue = true + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Start and stop execute instances automatically + + [[[parameter MaxHPCExecuteCoreCount]]] + Label = Max HPC Cores + Description = The total number of HPC execute cores to start + DefaultValue = 100 + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 1 + Config.IntegerOnly = true + + [[[parameter MaxHTCExecuteCoreCount]]] + Label = Max HTC Cores + Description = The total number of HTC execute cores to start + DefaultValue = 100 + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 1 + Config.IntegerOnly = true + + [[[parameter MaxDynamicExecuteCoreCount]]] + Label = Max Dyn Cores + Description = The total number of Dynamic execute cores to start + DefaultValue = 100 + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 1 + Config.IntegerOnly = true + + [[[parameter HPCMaxScalesetSize]]] + Label = Max VMs per VMSS + Description = The maximum number of VMs created per VM Scaleset e.g. switch in Slurm. + DefaultValue = 100 + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 1 + Config.IntegerOnly = true + + + [[[parameter HTCUseLowPrio]]] + Label = HTC Spot + DefaultValue = false + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Use Spot VMs for HTC execute hosts + + [[[parameter HTCSpotMaxPrice]]] + Label = Max Price HTC + DefaultValue = -1 + Description = Max price for Spot VMs in USD (value of -1 will not evict based on price) + Config.Plugin = pico.form.NumberTextBox + Conditions.Excluded := HTCUseLowPrio isnt true + Config.MinValue = -1 + + [[[parameter DynamicUseLowPrio]]] + Label = DynSpot + DefaultValue = false + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Use Spot VMs for Dynamic execute hosts + + [[[parameter DynamicSpotMaxPrice]]] + Label = Max Price Dyn + DefaultValue = -1 + Description = Max price for Spot VMs in USD (value of -1 will not evict based on price) + Config.Plugin = pico.form.NumberTextBox + Conditions.Excluded := DynamicUseLowPrio isnt true + Config.MinValue = -1 + + [[[parameter NumberLoginNodes]]] + Label = Num Login Nodes + DefaultValue = 0 + Description = Number of optional login nodes to create. + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 0 + Config.MaxValue = 10000 + Config.IntegerOnly = true + + [[parameters Networking]] + Order = 40 + + [[[parameter SubnetId]]] + Label = Subnet ID + Description = Subnet Resource Path (ResourceGroup/VirtualNetwork/Subnet) + ParameterType = Azure.Subnet + Required = True + + [[parameters High Availability]] + Order = 50 + Description = "Slurm can be setup in HA mode - where slurmctld is running on two nodes with failover. Note that checking this box will require an external NFS, so any reference to the 'builtin' NFS will be hidden." + [[[parameter configuration_slurm_ha_enabled]]] + Label = Slurm HA Node + Description = Deploy with an additional HA node + DefaultValue = false + ParameterType = Boolean + + +[parameters Network Attached Storage] +Order = 15 + + [[parameters Shared Storage]] + Order = 10 + + [[[parameter About Shared Storage]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''

The directories /sched and /shared are network attached mounts and exist on all nodes of the cluster.
+
+ Options for providing these mounts:
+ [Builtin]: The scheduler node is an NFS server that provides the mountpoint to the other nodes of the cluster (not supported for HA configurations).
+ [External NFS]: A network attached storage such as Azure Netapp Files, HPC Cache, or another VM running an NFS server provides the mountpoint.
+ [Azure Managed Lustre]: An Azure Managed Lustre deployment provides the mountpoint.
+

+

+ Note: the cluster must be terminated for changes to filesystem mounts to take effect. +

''' + Conditions.Hidden := false + + [[parameters Scheduler Mount]] + Order = 20 + Label = File-system Mount for /sched + + [[[parameter About sched]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''

Slurm's configuration is linked in from the /sched directory. It is managed by the scheduler node

''' + Order = 6 + + [[[parameter About sched part 2]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''

To disable the built-in NFS export of the /sched directory, and to use an external filesystem, select the checkbox below.

''' + Order = 7 + Conditions.Hidden := configuration_slurm_ha_enabled + + [[[parameter UseBuiltinSched]]] + Label = Use Builtin NFS + Description = Use the builtin NFS for /sched + DefaultValue = true + ParameterType = Boolean + Conditions.Hidden := configuration_slurm_ha_enabled + Disabled = configuration_slurm_ha_enabled + + [[[parameter NFSSchedDiskWarning]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template := "

Warning: switching an active cluster over to NFS or Lustre from Builtin will delete the shared disk.

" + Conditions.Hidden := UseBuiltinSched || configuration_slurm_ha_enabled + + [[[parameter NFSSchedType]]] + Label = FS Type + ParameterType = StringList + Config.Label = Type of shared filesystem to use for this cluster + Config.Plugin = pico.form.Dropdown + Config.Entries := {[Label="External NFS"; Value="nfs"], [Label="Azure Managed Lustre"; Value="lustre"]} + DefaultValue = nfs + Conditions.Hidden := UseBuiltinSched && !configuration_slurm_ha_enabled + + [[[parameter NFSSchedAddress]]] + Label = IP Address + Description = The IP address or hostname of the NFS server or Lustre FS. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + Config.ParameterType = String + Conditions.Hidden := UseBuiltinSched && !configuration_slurm_ha_enabled + + [[[parameter NFSSchedExportPath]]] + Label = Export Path + Description = The path exported by the file system + DefaultValue = /sched + Conditions.Hidden := UseBuiltinSched && !configuration_slurm_ha_enabled + + [[[parameter NFSSchedMountOptions]]] + Label = Mount Options + Description = NFS Client Mount Options + Conditions.Hidden := UseBuiltinSched && !configuration_slurm_ha_enabled + + + [[[parameter SchedFilesystemSize]]] + Label = Size (GB) + Description = The filesystem size (cannot be changed after initial start) + DefaultValue = 30 + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 10 + Config.MaxValue = 10240 + Config.IntegerOnly = true + Conditions.Excluded := !UseBuiltinSched || configuration_slurm_ha_enabled + + + + [[parameters Default NFS Share]] + Order = 30 + Label = File-system Mount for /shared + + [[[parameter About shared]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''

Users' home directories reside within the /shared mountpoint with the base homedir /shared/home.

''' + Order = 6 + + [[[parameter About shared part 2]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template = '''

To disable the built-in NFS export of the /shared directory, and to use an external filesystem, select the checkbox below.

''' + Order = 7 + Conditions.Hidden := configuration_slurm_ha_enabled + + [[[parameter UseBuiltinShared]]] + Label = Use Builtin NFS + Description = Use the builtin NFS for /share + DefaultValue = true + ParameterType = Boolean + Conditions.Hidden := configuration_slurm_ha_enabled + Disabled = configuration_slurm_ha_enabled + + [[[parameter NFSDiskWarning]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template := "

Warning: switching an active cluster over to NFS or Lustre from Builtin will delete the shared disk.

" + Conditions.Hidden := UseBuiltinShared || configuration_slurm_ha_enabled + + [[[parameter NFSType]]] + Label = FS Type + ParameterType = StringList + Config.Label = Type of shared filesystem to use for this cluster + Config.Plugin = pico.form.Dropdown + Config.Entries := {[Label="External NFS"; Value="nfs"], [Label="Azure Managed Lustre"; Value="lustre"]} + DefaultValue = nfs + Conditions.Hidden := UseBuiltinShared && !configuration_slurm_ha_enabled + + [[[parameter NFSAddress]]] + Label = IP Address + Description = The IP address or hostname of the NFS server or Lustre FS. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + Config.ParameterType = String + Conditions.Hidden := UseBuiltinShared && !configuration_slurm_ha_enabled + + [[[parameter NFSSharedExportPath]]] + Label = Export Path + Description = The path exported by the file system + DefaultValue = /shared + Conditions.Hidden := UseBuiltinShared && !configuration_slurm_ha_enabled + + [[[parameter NFSSharedMountOptions]]] + Label = Mount Options + Description = NFS Client Mount Options + Conditions.Hidden := UseBuiltinShared && !configuration_slurm_ha_enabled + + + [[[parameter FilesystemSize]]] + Label = Size (GB) + Description = The filesystem size (cannot be changed after initial start) + DefaultValue = 100 + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 10 + Config.MaxValue = 10240 + Config.IntegerOnly = true + Conditions.Excluded := !UseBuiltinShared || configuration_slurm_ha_enabled + + [[parameters Additional NFS Mount]] + Order = 40 + Label = Additional Filesystem Mount + [[[parameter Additional Shared FS Mount Readme]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template := "

Mount another shared filesystem endpoint on the cluster nodes.

" + Order = 20 + + [[[parameter AdditionalNFS1]]] + HideLabel = true + DefaultValue = false + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Add Shared Filesystem mount + + [[[parameter AdditionalNFSType1]]] + Label = FS Type + ParameterType = StringList + Config.Label = Shared filesystem type of the additional mount + Config.Plugin = pico.form.Dropdown + Config.Entries := {[Label="External NFS"; Value="nfs"], [Label="Azure Managed Lustre"; Value="lustre"]} + DefaultValue = nfs + Conditions.Excluded := AdditionalNFS1 isnt true + + [[[parameter AdditionalNFSAddress1]]] + Label = IP Address + Description = The IP address or hostname of the additional mount. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + Config.ParameterType = String + Conditions.Excluded := AdditionalNFS1 isnt true + + [[[parameter AdditionalNFSMountPoint1]]] + Label = Mount Point + Description = The path at which to mount the Filesystem + DefaultValue = /data + Conditions.Excluded := AdditionalNFS1 isnt true + + [[[parameter AdditionalNFSExportPath1]]] + Label = Export Path + Description = The path exported by the file system + DefaultValue = /anf-vol1/wrf/data + Conditions.Excluded := AdditionalNFS1 isnt true + + [[[parameter AdditionalNFSMountOptions1]]] + Label = Mount Options + Description = Filesystem Client Mount Options + Conditions.Excluded := AdditionalNFS1 isnt true + + [[[parameter AdditionalNFS2]]] + HideLabel = true + DefaultValue = false + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Add Shared Filesystem mount + + [[[parameter AdditionalNFSType2]]] + Label = FS Type + ParameterType = StringList + Config.Label = Shared filesystem type of the additional mount + Config.Plugin = pico.form.Dropdown + Config.Entries := {[Label="External NFS"; Value="nfs"], [Label="Azure Managed Lustre"; Value="lustre"]} + DefaultValue = nfs + Conditions.Excluded := AdditionalNFS2 isnt true + + [[[parameter AdditionalNFSAddress2]]] + Label = IP Address + Description = The IP address or hostname of the additional mount. Also accepts a list comma-separated addresses, for example, to mount a frontend load-balanced Azure HPC Cache. + Config.ParameterType = String + Conditions.Excluded := AdditionalNFS2 isnt true + + [[[parameter AdditionalNFSMountPoint2]]] + Label = Mount Point + Description = The path at which to mount the Filesystem + DefaultValue = /apps + Conditions.Excluded := AdditionalNFS2 isnt true + + [[[parameter AdditionalNFSExportPath2]]] + Label = Export Path + Description = The path exported by the file system + DefaultValue = /anf-vol1/wrf/apps + Conditions.Excluded := AdditionalNFS2 isnt true + + [[[parameter AdditionalNFSMountOptions2]]] + Label = Mount Options + Description = Filesystem Client Mount Options + Conditions.Excluded := AdditionalNFS2 isnt true + + +[parameters Advanced Settings] +Order = 20 + + [[parameters Azure Settings]] + Order = 10 + + [[[parameter Credentials]]] + Description = The credentials for the cloud provider + ParameterType = Cloud.Credentials + + [[[parameter ManagedIdentity]]] + Label = Managed Id + Description = Optionally assign an Azure user assigned managed identity to all nodes to access Azure resources using assigned roles. + ParameterType = Azure.ManagedIdentity + DefaultValue = =undefined + + [[[parameter BootDiskSize]]] + Description = Optional: Size of the OS/boot disk in GB for all nodes in the cluster (leave at 0 to use Image size) + ParameterType = Integer + Config.Plugin = pico.form.NumberTextBox + Config.MinValue = 0 + Config.MaxValue = 32,000 + Config.IntegerOnly = true + Config.Increment = 64 + DefaultValue = 0 + + [[parameters Slurm Settings ]] + + Order = 5 + + [[[parameter slurm_version_warning]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Config.Template := "
Note: For SLES HPC, we can only install the version supported by SLES HPC's zypper repos. At the time of this release, that is 23.02.7
" + + + [[[parameter configuration_slurm_version]]] + Required = True + Label = Slurm Version + Description = Version of Slurm to install on the cluster + ParameterType = StringList + Config.Plugin = pico.form.Dropdown + Config.FreeForm = true + Config.Entries := {[Value="23.02.7-4"], [Value="23.11.7-1"]} + DefaultValue = 23.11.7-1 + + [[[parameter configuration_slurm_accounting_enabled]]] + Label = Job Accounting + DefaultValue = false + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Configure Slurm job accounting + + [[[parameter slurm_database_warning]]] + HideLabel = true + Config.Plugin = pico.widget.HtmlTemplateWidget + Conditions.Excluded := configuration_slurm_accounting_enabled isnt true + Config.Template := "
Note: Checking this box will create persistent databases and tables in SQL DB provided. Deleting this cluster will not automatically delete those databases. User is responsible for periodically purging/archiving their slurm databases to maintain costs.
" + + [[[parameter configuration_slurm_accounting_url]]] + Label = Slurm DBD URL + Description = URL of the database to use for Slurm job accounting + Conditions.Excluded := configuration_slurm_accounting_enabled isnt true + + [[[parameter configuration_slurm_accounting_storageloc]]] + Label = Database name + Description = Database name to store slurm accounting records + Conditions.Excluded := configuration_slurm_accounting_enabled isnt true + + [[[parameter configuration_slurm_accounting_user]]] + Label = Slurm DBD User + Description = User for Slurm DBD admin + Conditions.Excluded := configuration_slurm_accounting_enabled isnt true + + [[[parameter configuration_slurm_accounting_password]]] + Label = Slurm DBD Password + Description = Password for Slurm DBD admin + ParameterType = Password + Conditions.Excluded := configuration_slurm_accounting_enabled isnt true + + [[[parameter configuration_slurm_accounting_certificate_url]]] + Label = SSL Certificate URL + Description = URL to fetch SSL certificate for authentication to DB. Use AzureCA.pem (embedded) for use with deprecated MariaDB instances. + Conditions.Excluded := configuration_slurm_accounting_enabled isnt true + ParameterType = StringList + Config.Plugin = pico.form.Dropdown + Config.FreeForm = true + Config.Entries := {[Value=""], [Value="AzureCA.pem"]} + DefaultValue = "" + + [[[parameter configuration_slurm_shutdown_policy]]] + Label = Shutdown Policy + description = By default, autostop will Delete stopped VMS for lowest cost. Optionally, Stop/Deallocate the VMs for faster restart instead. + DefaultValue = Terminate + config.plugin = pico.control.AutoCompleteDropdown + [[[[list Config.Entries]]]] + Name = Terminate + Label = Terminate + [[[[list Config.Entries]]]] + Name = Deallocate + Label = Deallocate + + [[[parameter EnableTerminateNotification]]] + Label = Enable Termination notifications + DefaultValue = False + + [[[parameter additional_slurm_config]]] + Label = Slurm Configuration + Description = Any additional lines to add to slurm.conf + ParameterType = Text + + [[[parameter configuration_slurm_launch_parameters]]] + Label = Launch Parameters + Description = Deploy Slurm with Launch Parameters (comma delimited) + DefaultValue = '' + ParameterType = String + + + + [[parameters Software]] + Description = "Specify the scheduling software, and base OS installed on all nodes, and optionally the cluster-init and chef versions from your locker." + Order = 10 + + [[[parameter NodeNameIsHostname]]] + Label = Name As Hostname + Description = Should the hostname match the nodename for execute nodes? + ParameterType = Boolean + DefaultValue = true + + [[[parameter NodeNamePrefix]]] + Label = Node Prefix + Description = Prefix for generated node names, i.e. "prefix-" generates prefix-nodearray-1. Use 'Cluster Prefix' to get $ClusterName-nodearray-1 + ParameterType = StringList + Config.Plugin = pico.form.Dropdown + Config.FreeForm = true + DefaultValue = "Cluster Prefix" + Config.Entries := {[Value=""], [Value="Cluster Prefix"]} + Conditions.Hidden := NodeNameIsHostname != true + + [[[parameter SchedulerHostName]]] + Label = Scheduler Hostname + Description = Hostname of scheduler. 'Generated' uses the default generated hostname. 'Cluster Prefix' will generate $ClusterName-scheduler. + ParameterType = StringList + Config.Plugin = pico.form.Dropdown + Config.FreeForm = true + DefaultValue = "Cluster Prefix" + Config.Entries := {[Value="Generated"], [Value="Cluster Prefix"]} + Conditions.Hidden := NodeNameIsHostname != true + + [[[parameter SchedulerImageName]]] + Label = Scheduler OS + ParameterType = Cloud.Image + Config.OS = linux + DefaultValue = almalinux8 + Config.Filter := Package in {"cycle.image.centos7", "cycle.image.ubuntu20", "cycle.image.ubuntu22", "cycle.image.sles15-hpc", "almalinux8"} + + [[[parameter HPCImageName]]] + Label = HPC OS + ParameterType = Cloud.Image + Config.OS = linux + DefaultValue = almalinux8 + Config.Filter := Package in {"cycle.image.centos7", "cycle.image.ubuntu20", "cycle.image.ubuntu22", "cycle.image.sles15-hpc", "almalinux8"} + + [[[parameter HTCImageName]]] + Label = HTC OS + ParameterType = Cloud.Image + Config.OS = linux + DefaultValue = almalinux8 + Config.Filter := Package in {"cycle.image.centos7", "cycle.image.ubuntu20", "cycle.image.ubuntu22", "cycle.image.sles15-hpc", "almalinux8"} + + [[[parameter DynamicImageName]]] + Label = Dynamic OS + ParameterType = Cloud.Image + Config.OS = linux + DefaultValue = almalinux8 + Config.Filter := Package in {"cycle.image.centos7", "cycle.image.ubuntu20", "cycle.image.ubuntu22", "cycle.image.sles15-hpc", "almalinux8"} + + [[[parameter SchedulerClusterInitSpecs]]] + Label = Scheduler Cluster-Init + DefaultValue = =undefined + Description = Cluster init specs to apply to the scheduler node + ParameterType = Cloud.ClusterInitSpecs + + [[[parameter HTCClusterInitSpecs]]] + Label = HTC Cluster-Init + DefaultValue = =undefined + Description = Cluster init specs to apply to HTC execute nodes + ParameterType = Cloud.ClusterInitSpecs + + [[[parameter HPCClusterInitSpecs]]] + Label = HPC Cluster-Init + DefaultValue = =undefined + Description = Cluster init specs to apply to HPC execute nodes + ParameterType = Cloud.ClusterInitSpecs + + [[[parameter DynamicClusterInitSpecs]]] + Label = Dyn Cluster-Init + DefaultValue = =undefined + Description = Cluster init specs to apply to Dynamic execute nodes + ParameterType = Cloud.ClusterInitSpecs + + [[[parameter configuration_slurm_disable_pmc]]] + Label = Disable PMC + Description = Disable packages from packages.microsoft.com + ParameterType = Boolean + DefaultValue = false + + + [[parameters Advanced Networking]] + + [[[parameter ReturnProxy]]] + Label = Return Proxy + DefaultValue = true + ParameterType = Boolean + Config.Label = Use SSH tunnel to connect to CycleCloud (required if direct access is blocked) + + [[[parameter UsePublicNetwork]]] + Label = Public Head Node + DefaultValue = true + ParameterType = Boolean + Config.Label = Access scheduler node from the Internet + + [[[parameter ExecuteNodesPublic]]] + Label = Public Execute + DefaultValue = false + ParameterType = Boolean + Config.Label = Access execute nodes from the Internet + Conditions.Excluded := UsePublicNetwork isnt true + + [[[parameter SchedulerZone]]] + Label = Scheduler Zone + Description = The availability zone in which to deploy the scheduler node. + DefaultValue = =undefined + Config.Plugin = pico.form.Dropdown + Config.Entries := {[Value=1], [Value=2], [Value=3], [Value=undefined; Label="Any"]} + + [[[parameter SchedulerHAZone]]] + Label = Scheduler HA Zone + Description = The availability zone in which to deploy the scheduler-ha node. + DefaultValue = =undefined + Config.Plugin = pico.form.Dropdown + Config.Entries := {[Value=1], [Value=2], [Value=3], [Value=undefined; Label="Any"]} + Conditions.Hidden := configuration_slurm_ha_enabled isnt true + + [[parameters Node Health Checks]] + Description = "Section for configuring Node Health Checks" + Order = 12 + + [[[parameter EnableNodeHealthChecks]]] + Label = Enable NHC tests + DefaultValue = false + Widget.Plugin = pico.form.BooleanCheckBox + Widget.Label = Run Node Health Checks on startup diff --git a/apps/wrf/download_gfs_files.py b/apps/wrf/download_gfs_files.py new file mode 100644 index 000000000..3eacddda0 --- /dev/null +++ b/apps/wrf/download_gfs_files.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python +################################################################# +# Python Script to retrieve 93 online Data files of 'ds084.1', +# total 20.11G. This script uses 'requests' to download data. +# +# Highlight this script by Select All, Copy and Paste it into a file; +# make the file executable and run it on command line. +# +# You need pass in your password as a parameter to execute +# this script; or you can set an environment variable RDAPSWD +# if your Operating System supports it. +# +# Contact rdahelp@ucar.edu (RDA help desk) for further assistance. +################################################################# + + +import sys, os +import requests + +def check_file_status(filepath, filesize): + sys.stdout.write('\r') + sys.stdout.flush() + size = int(os.stat(filepath).st_size) + percent_complete = (size/filesize)*100 + sys.stdout.write('%.3f %s' % (percent_complete, '% Completed')) + sys.stdout.flush() + +# Try to get password +if len(sys.argv) < 2 and not 'RDAPSWD' in os.environ: + try: + import getpass + input = getpass.getpass + except: + try: + input = raw_input + except: + pass + pswd = input('Password: ') +else: + try: + pswd = sys.argv[1] + except: + pswd = os.environ['RDAPSWD'] + +url = 'https://rda.ucar.edu/cgi-bin/login' +values = {'email' : 'enter-your-email-here@email.com', 'passwd' : pswd, 'action' : 'login'} +# Authenticate +ret = requests.post(url,data=values) +if ret.status_code != 200: + print('Bad Authentication') + print(ret.text) + exit(1) +dspath = 'https://rda.ucar.edu/data/ds084.1/' +filelist = [ +'2018/20180617/gfs.0p25.2018061700.f000.grib2', +'2018/20180617/gfs.0p25.2018061700.f003.grib2', +'2018/20180617/gfs.0p25.2018061700.f006.grib2', +'2018/20180617/gfs.0p25.2018061700.f009.grib2', +'2018/20180617/gfs.0p25.2018061700.f012.grib2', +'2018/20180617/gfs.0p25.2018061700.f015.grib2', +'2018/20180617/gfs.0p25.2018061700.f018.grib2', +'2018/20180617/gfs.0p25.2018061700.f021.grib2', +'2018/20180617/gfs.0p25.2018061700.f024.grib2', +'2018/20180617/gfs.0p25.2018061700.f027.grib2', +'2018/20180617/gfs.0p25.2018061700.f030.grib2', +'2018/20180617/gfs.0p25.2018061700.f033.grib2', +'2018/20180617/gfs.0p25.2018061700.f036.grib2', +'2018/20180617/gfs.0p25.2018061700.f039.grib2', +'2018/20180617/gfs.0p25.2018061700.f042.grib2', +'2018/20180617/gfs.0p25.2018061700.f045.grib2', +'2018/20180617/gfs.0p25.2018061700.f048.grib2', +'2018/20180617/gfs.0p25.2018061700.f051.grib2', +'2018/20180617/gfs.0p25.2018061700.f054.grib2', +'2018/20180617/gfs.0p25.2018061700.f057.grib2', +'2018/20180617/gfs.0p25.2018061700.f060.grib2', +'2018/20180617/gfs.0p25.2018061700.f063.grib2', +'2018/20180617/gfs.0p25.2018061700.f066.grib2', +'2018/20180617/gfs.0p25.2018061700.f069.grib2', +'2018/20180617/gfs.0p25.2018061700.f072.grib2', +'2018/20180617/gfs.0p25.2018061700.f075.grib2', +'2018/20180617/gfs.0p25.2018061700.f078.grib2', +'2018/20180617/gfs.0p25.2018061700.f081.grib2', +'2018/20180617/gfs.0p25.2018061700.f084.grib2', +'2018/20180617/gfs.0p25.2018061700.f087.grib2', +'2018/20180617/gfs.0p25.2018061700.f090.grib2', +'2018/20180617/gfs.0p25.2018061700.f093.grib2', +'2018/20180617/gfs.0p25.2018061700.f096.grib2', +'2018/20180617/gfs.0p25.2018061700.f099.grib2', +'2018/20180617/gfs.0p25.2018061700.f102.grib2', +'2018/20180617/gfs.0p25.2018061700.f105.grib2', +'2018/20180617/gfs.0p25.2018061700.f108.grib2', +'2018/20180617/gfs.0p25.2018061700.f111.grib2', +'2018/20180617/gfs.0p25.2018061700.f114.grib2', +'2018/20180617/gfs.0p25.2018061700.f117.grib2', +'2018/20180617/gfs.0p25.2018061700.f120.grib2', +'2018/20180617/gfs.0p25.2018061700.f123.grib2', +'2018/20180617/gfs.0p25.2018061700.f126.grib2', +'2018/20180617/gfs.0p25.2018061700.f129.grib2', +'2018/20180617/gfs.0p25.2018061700.f132.grib2', +'2018/20180617/gfs.0p25.2018061700.f135.grib2', +'2018/20180617/gfs.0p25.2018061700.f138.grib2', +'2018/20180617/gfs.0p25.2018061700.f141.grib2', +'2018/20180617/gfs.0p25.2018061700.f144.grib2', +'2018/20180617/gfs.0p25.2018061700.f147.grib2', +'2018/20180617/gfs.0p25.2018061700.f150.grib2', +'2018/20180617/gfs.0p25.2018061700.f153.grib2', +'2018/20180617/gfs.0p25.2018061700.f156.grib2', +'2018/20180617/gfs.0p25.2018061700.f159.grib2', +'2018/20180617/gfs.0p25.2018061700.f162.grib2', +'2018/20180617/gfs.0p25.2018061700.f165.grib2', +'2018/20180617/gfs.0p25.2018061700.f168.grib2', +'2018/20180617/gfs.0p25.2018061700.f171.grib2', +'2018/20180617/gfs.0p25.2018061700.f174.grib2', +'2018/20180617/gfs.0p25.2018061700.f177.grib2', +'2018/20180617/gfs.0p25.2018061700.f180.grib2', +'2018/20180617/gfs.0p25.2018061700.f183.grib2', +'2018/20180617/gfs.0p25.2018061700.f186.grib2', +'2018/20180617/gfs.0p25.2018061700.f189.grib2', +'2018/20180617/gfs.0p25.2018061700.f192.grib2', +'2018/20180617/gfs.0p25.2018061700.f195.grib2', +'2018/20180617/gfs.0p25.2018061700.f198.grib2', +'2018/20180617/gfs.0p25.2018061700.f201.grib2', +'2018/20180617/gfs.0p25.2018061700.f204.grib2', +'2018/20180617/gfs.0p25.2018061700.f207.grib2', +'2018/20180617/gfs.0p25.2018061700.f210.grib2', +'2018/20180617/gfs.0p25.2018061700.f213.grib2', +'2018/20180617/gfs.0p25.2018061700.f216.grib2', +'2018/20180617/gfs.0p25.2018061700.f219.grib2', +'2018/20180617/gfs.0p25.2018061700.f222.grib2', +'2018/20180617/gfs.0p25.2018061700.f225.grib2', +'2018/20180617/gfs.0p25.2018061700.f228.grib2', +'2018/20180617/gfs.0p25.2018061700.f231.grib2', +'2018/20180617/gfs.0p25.2018061700.f234.grib2', +'2018/20180617/gfs.0p25.2018061700.f237.grib2', +'2018/20180617/gfs.0p25.2018061700.f240.grib2', +'2018/20180617/gfs.0p25.2018061700.f252.grib2', +'2018/20180617/gfs.0p25.2018061700.f264.grib2', +'2018/20180617/gfs.0p25.2018061700.f276.grib2', +'2018/20180617/gfs.0p25.2018061700.f288.grib2', +'2018/20180617/gfs.0p25.2018061700.f300.grib2', +'2018/20180617/gfs.0p25.2018061700.f312.grib2', +'2018/20180617/gfs.0p25.2018061700.f324.grib2', +'2018/20180617/gfs.0p25.2018061700.f336.grib2', +'2018/20180617/gfs.0p25.2018061700.f348.grib2', +'2018/20180617/gfs.0p25.2018061700.f360.grib2', +'2018/20180617/gfs.0p25.2018061700.f372.grib2', +'2018/20180617/gfs.0p25.2018061700.f384.grib2'] +for file in filelist: + filename=dspath+file + file_base = os.path.basename(file) + print('Downloading',file_base) + req = requests.get(filename, cookies = ret.cookies, allow_redirects=True, stream=True) + filesize = int(req.headers['Content-length']) + with open(file_base, 'wb') as outfile: + chunk_size=1048576 + for chunk in req.iter_content(chunk_size=chunk_size): + outfile.write(chunk) + if chunk_size < filesize: + check_file_status(file_base, filesize) + check_file_status(file_base, filesize) + print() diff --git a/apps/wrf/env-variables b/apps/wrf/env-variables new file mode 100644 index 000000000..efa06ae00 --- /dev/null +++ b/apps/wrf/env-variables @@ -0,0 +1,28 @@ +SKU_TYPE=${1:-$hbv3} +SKU_TYPE=${SKU_TYPE:-$hbv3} +SHARED_APP=${2:-/apps} +SHARED_APP=${SHARED_APP:-/apps} + +echo "SKU_TYPE:" $SKU_TYPE +echo "SHARED_APP:" $SHARED_APP + +if ! rpm -q python3; then + sudo yum install -y python3 + pip install wget +fi +source /etc/profile.d/modules.sh +export MODULEPATH=${SHARED_APP}/modulefiles/${SKU_TYPE}:$MODULEPATH +module use ${SHARED_APP}/modulefiles +module load spack/spack +source $SPACK_SETUP_ENV +spack load netcdf-fortran^openmpi +spack load hdf5^openmpi +spack load perl +module use /usr/share/Modules/modulefiles +module load mpi/openmpi-5.0.2 +module load gcc-9.2.0 +module load wrf/4.1.5-openmpi +mpi_options="-x LD_LIBRARY_PATH " +if [ -n $LD_PRELOAD ]; then + mpi_options+="-x LD_PRELOAD" +fi diff --git a/apps/wrf/images/logo-genoma.jpeg b/apps/wrf/images/logo-genoma.jpeg new file mode 100644 index 000000000..ace7df9b6 Binary files /dev/null and b/apps/wrf/images/logo-genoma.jpeg differ diff --git a/apps/wrf/images/wrf-logo.jpg b/apps/wrf/images/wrf-logo.jpg new file mode 100644 index 000000000..76d0386a8 Binary files /dev/null and b/apps/wrf/images/wrf-logo.jpg differ diff --git a/apps/wrf/images/wrf-test-results.png b/apps/wrf/images/wrf-test-results.png new file mode 100644 index 000000000..d98770d28 Binary files /dev/null and b/apps/wrf/images/wrf-test-results.png differ diff --git a/apps/wrf/rda-download-gfs-files.py b/apps/wrf/rda-download-gfs-files.py new file mode 100644 index 000000000..63d690c2d --- /dev/null +++ b/apps/wrf/rda-download-gfs-files.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python +""" +Python script to download selected files from rda.ucar.edu. +After you save the file, don't forget to make it executable +i.e. - "chmod 755 " +""" +import sys, os +from urllib.request import build_opener + +opener = build_opener() + +filelist = [ + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f000.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f003.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f006.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f009.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f012.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f015.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f018.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f021.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f024.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f027.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f030.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f033.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f036.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f039.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f042.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f045.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f048.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f051.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f054.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f057.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f060.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f063.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f066.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f069.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f072.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f075.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f078.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f081.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f084.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f087.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f090.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f093.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f096.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f099.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f102.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f105.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f108.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f111.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f114.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f117.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f120.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f123.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f126.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f129.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f132.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f135.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f138.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f141.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f144.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f147.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f150.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f153.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f156.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f159.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f162.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f165.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f168.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f171.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f174.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f177.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f180.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f183.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f186.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f189.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f192.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f195.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f198.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f201.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f204.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f207.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f210.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f213.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f216.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f219.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f222.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f225.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f228.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f231.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f234.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f237.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f240.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f252.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f264.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f276.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f288.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f300.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f312.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f324.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f336.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f348.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f360.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f372.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061700.f384.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f000.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f003.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f006.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f009.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f012.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f015.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f018.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f021.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f024.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f027.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f030.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f033.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f036.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f039.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f042.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f045.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f048.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f051.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f054.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f057.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f060.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f063.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f066.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f069.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f072.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f075.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f078.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f081.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f084.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f087.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f090.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f093.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f096.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f099.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f102.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f105.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f108.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f111.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f114.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f117.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f120.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f123.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f126.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f129.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f132.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f135.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f138.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f141.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f144.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f147.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f150.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f153.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f156.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f159.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f162.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f165.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f168.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f171.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f174.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f177.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f180.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f183.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f186.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f189.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f192.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f195.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f198.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f201.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f204.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f207.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f210.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f213.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f216.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f219.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f222.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f225.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f228.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f231.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f234.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f237.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f240.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f252.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f264.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f276.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f288.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f300.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f312.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f324.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f336.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f348.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f360.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f372.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061706.f384.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f000.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f003.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f006.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f009.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f012.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f015.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f018.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f021.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f024.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f027.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f030.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f033.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f036.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f039.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f042.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f045.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f048.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f051.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f054.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f057.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f060.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f063.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f066.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f069.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f072.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f075.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f078.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f081.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f084.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f087.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f090.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f093.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f096.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f099.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f102.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f105.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f108.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f111.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f114.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f117.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f120.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f123.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f126.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f129.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f132.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f135.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f138.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f141.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f144.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f147.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f150.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f153.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f156.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f159.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f162.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f165.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f168.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f171.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f174.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f177.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f180.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f183.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f186.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f189.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f192.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f195.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f198.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f201.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f204.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f207.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f210.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f213.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f216.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f219.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f222.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f225.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f228.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f231.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f234.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f237.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f240.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f252.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f264.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f276.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f288.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f300.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f312.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f324.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f336.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f348.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f360.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f372.grib2', + 'https://data.rda.ucar.edu/ds084.1/2018/20180617/gfs.0p25.2018061712.f384.grib2' +] + +for file in filelist: + ofile = os.path.basename(file) + sys.stdout.write("downloading " + ofile + " ... ") + sys.stdout.flush() + infile = opener.open(file) + outfile = open(ofile, "wb") + outfile.write(infile.read()) + outfile.close() + sys.stdout.write("done\n") diff --git a/apps/wrf/readme.md b/apps/wrf/readme.md index e748a6341..29dffd43b 100755 --- a/apps/wrf/readme.md +++ b/apps/wrf/readme.md @@ -1,110 +1,362 @@ -## Install and run WRF v4 and WPS v4 - +# Install and run WPS and WRF v4 - Setup guide + ## Prerequisites -Cluster is built with the desired configuration for networking, storage, compute etc. You can see the tutorial or examples folder in this repo for how to set this up. Spack is installed (See [here](../spack/readme.md) for details). +- You need a cluster built with the desired configuration for networking, storage, compute etc. You can use the tutorial in this repo with an end-to-end instructions to setup a lab environment on Cycle Cloud to run WPS and WRF v4. (See [Install and run WPS and WRF v4 - end-to-end setup guide](../../experimental/wrf_on_cyclecloud/readme.md) for details). +- As this procedure uses HBv3 VMs to run WRFv4 simulations, you may need to request quota increase for this type of SKU in the subscription and region you will deploy the environment. You can use different SKU if you want to. +- You need to download/clone the azurehpc GitHub repository +- This installation procedure requires you have 2 folders mounted /apps and /data on your storage solution + +### Summary of steps: +- Install WPS/WRF v4 software (from “azurehpc” scripts) +- Download data for WRF v4 +- Edit data locations in WRF config files +- Generate WRF v4 input files, change permissions +- Run WRF v4 applications for testing -Dependencies for binary version: +### Install WPS/WRF v4 software (via “azurehpc” scripts) -* None +Spin up and SSH to a Worker Node VM (HBv3). +**Important 1**: You must have the /apps and /data volumes correctly mounted on head and worker nodes. It is required for WRF setup scripts. -First copy the apps directory to the cluster in a shared directory. The `azhpc-scp` can be used to do this: +**Important 2**: You need to be root user to run all commands below. +Be sure you have downloaded azurehpc GitHub repository in /data folder: ``` -azhpc-scp -r $azhpc_dir/apps/. hpcuser@headnode:. +cd /data +git clone https://github.com/marcusgaspar/azurehpc.git +#git clone https://github.com/Azure/azurehpc.git ``` -> Alternatively you can checkout the azurehpc repository but you will need to update the paths according to where you put it. +Then, run the following commands: +``` +# need to be root user for building everything +sudo su - +###### Setup Spack +cd /data/azurehpc/apps/spack +./build_spack.sh hbv3 +source /apps/spack/0.16.3/spack/share/spack/setup-env.sh -## Connect to the headnode +###### Setup WRF +# MPI_TYPE : openmpi or mvapich2 +# SKU_TYPE : hb, hbv2, hc, hbv3 +# This procedure uses openmpi and hbv3 +cd /data/azurehpc/apps/wrf/ +./build_wrf.sh openmpi hbv3 +./build_wps.sh openmpi hbv3 ``` -azhpc-connect -u hpcuser headnode + +Run the command to source the env-variables file: ``` +###### Source Variables +# Keep as root +#sudo su - +source /data/azurehpc/apps/wrf/env-variables hbv3 +``` + +### Download WFR Input DATA (new_conus2.5km) +WPS (WRF preprocessing system) is used to create WRF input cases. WRF v3 models are not compatible with WRF v4, so some WRF v4 input cases will need to be generated with WPS v4. +I will outline the procedure used to create a new_conus2.5km input case for WRF v4. + +References: +- [azurehpc/apps/wrf](https://www2.mmm.ucar.edu/wrf/users/download/get_sources_wps_geog.html) +- [Static data downloads from ucar.edu](https://www2.mmm.ucar.edu/wrf/users/download/get_sources_wps_geog.html) -> Note : It's important to be connected with the hpcuser which have a shared home, versus hpcadmin which doesn't and can't run pbs jobs +Run the commands below to download geopraphical static data for WPS v4: +``` +# Keep as root +#sudo su - -## Installation +mkdir /data/wrfdata +cd /data/wrfdata +git clone https://github.com/akirakyle/WRF_benchmarks.git +wget https://www2.mmm.ucar.edu/wrf/src/wps_files/topo_10m.tar.bz2 +wget https://www2.mmm.ucar.edu/wrf/src/wps_files/topo_2m.tar.bz2 +wget https://www2.mmm.ucar.edu/wrf/src/wps_files/topo_30s.tar.bz2 +wget https://www2.mmm.ucar.edu/wrf/src/wps_files/albedo_modis.tar.bz2 +wget https://www2.mmm.ucar.edu/wrf/src/wps_files/maxsnowalb_modis.tar.bz2 +wget https://www2.mmm.ucar.edu/wrf/src/wps_files/topo_gmted2010_30s.tar.bz2 +wget https://www2.mmm.ucar.edu/wrf/src/wps_files/modis_landuse_20class_30s_with_lakes.tar.bz2 +wget https://www2.mmm.ucar.edu/wrf/src/wps_files/soiltemp_1deg.tar.bz2 +wget https://www2.mmm.ucar.edu/wrf/src/wps_files/soiltype_top_30s.tar.bz2 +wget https://www2.mmm.ucar.edu/wrf/src/wps_files/soiltype_bot_30s.tar.bz2 +wget https://www2.mmm.ucar.edu/wrf/src/wps_files/greenfrac_fpar_modis.tar.bz2 +wget https://www2.mmm.ucar.edu/wrf/src/wps_files/lai_modis_10m.tar.bz2 +wget https://www2.mmm.ucar.edu/wrf/src/wps_files/orogwd_10m.tar.bz2 -### Run wrf install script +tar xjvf topo_10m.tar.bz2 +tar xjvf topo_2m.tar.bz2 +tar xjvf topo_30s.tar.bz2 +tar xjvf albedo_modis.tar.bz2 +tar xjvf maxsnowalb_modis.tar.bz2 +tar xjvf topo_gmted2010_30s.tar.bz2 +tar xjvf modis_landuse_20class_30s_with_lakes.tar.bz2 +tar xjvf soiltemp_1deg.tar.bz2 +tar xjvf soiltype_top_30s.tar.bz2 +tar xjvf soiltype_bot_30s.tar.bz2 +tar xjvf greenfrac_fpar_modis.tar.bz2 +tar xjvf lai_modis_10m.tar.bz2 +tar xjvf orogwd_10m.tar.bz2 ``` -# MPI_TYPE : openmpi or mvapich2 -# SKU_TYPE : hb, hbv2, hc -# OMP : leave empty for none or use omp as a value -apps/wrf/build_wrf.sh + +Download the data from https://rda.ucar.edu/datasets/ds084.1/ +1) Register to the site and go to the "Data Access" section and download the following data: +2) Data Access > Web File Listing > Complete File List > 2018 > 2018-06-17 +3) Select: from gfs.0p25.2018061700.f000.grib2 to gfs.0p25.2018061712.f384.grib2 (93 files around 20GB) +4) Select Python Download Script and upload it to the worker VM as download_gfs_files.py. + +Run the commands and be sure of copying the download_gfs_files.py the folder below: ``` +# Keep as root +#sudo su - -Run the WPS installation script if you need to install WPS (WRF needs to be installed first) +mkdir /data/wrfdata/gfs_files +cd /data/wrfdata/gfs_files +python download_gfs_files.py ``` -# MPI_TYPE : openmpi or mvapich2 -# SKU_TYPE : hb, hbv2, hc -apps/wrf/build_wps.sh + +### Generate WRF4 Input Files +Modify your **namelist.wps** file, setting the correct paths for **geog_data_path**, **opt_geogrid_tbl_path** and **opt_metgrid_tbl_path**: ``` +# Keep as root +#sudo su - -## Running +#### Change Data Locations +cd /apps/hbv3/wps-openmpi/WPS-4.1 +cp namelist.wps namelist.wps.old +cp /data/wrfdata/WRF_benchmarks/cases/new_conus2.5km/namelist.wps namelist.wps +vi /apps/hbv3/wps-openmpi/WPS-4.1/namelist.wps + geog_data_path = '/data/wrfdata/', + opt_geogrid_tbl_path = '/apps/hbv3/wps-openmpi/WPS-4.1/geogrid/', + opt_metgrid_tbl_path = '/apps/hbv3/wps-openmpi/WPS-4.1/metgrid/', +``` -Now, you can run wrf as follows: +### Run Applications: +#### Run geogrid.exe and metgrid.exe ``` -qsub -l select=2:ncpus=60:mpiprocs=15,place=scatter:excl -v SKU_TYPE=hb,INPUTDIR=/path/to/inputfiles apps/wrf/run_wrf_openmpi.pbs +# Keep as root +sudo su - +# Source variables +source /data/azurehpc/apps/wrf/env-variables hbv3 + +#### Run geogrid.exe +cd /apps/hbv3/wps-openmpi/WPS-4.1/ +mpirun --allow-run-as-root ./geogrid.exe +ln -s ungrib/Variable_Tables/Vtable.GFS Vtable +./link_grib.csh /data/wrfdata/gfs_files/gfs.0p25.20180617* +./ungrib.exe >& ungrib.log + +#### Run metgrid.exe +mpirun --allow-run-as-root ./metgrid.exe ``` -> Where SKU_TYPE is the sku type you are running on and INPUTDIR contains the location of wrf input files (namelist.input, wrfbdy_d01 and wrfinput_d01) -## How to create WRF input cases with WPS -WPS (WRF preprocessing system) is used to create WRF input cases. WRF v3 models are not compatible with WRF v4, so some WRF v4 input cases will need to be generated with WSP v4. -I will outline the procedure used to create a new_conus2.5km input case for WRF v4. +Expected results 1: +``` +# ! Successful completion of metgrid. ! +# You may receive also some warnings, which is expected: +Note: The following floating-point exceptions are signalling: IEEE_OVERFLOW_FLAG IEEE_UNDERFLOW_FLAG IEEE_DENORMAL +``` +Expected results 2: +You will find the files below: +``` +ls -l *met_em.d* +# Should see the following files: +# met_em.d01.2018-06-17_12:00:00.nc +# met_em.d01.2018-06-17_09:00:00.nc +# met_em.d01.2018-06-17_06:00:00.nc +# met_em.d01.2018-06-17_03:00:00.nc +# met_em.d01.2018-06-17_00:00:00.nc +``` -Git clone the following repository to get access to the WRF benchmarks cases namelist.input and namelist.wps files (which will be needed later as WPS input and an master input file for WRF). +#### Run real.exe +``` +# Keep as root +sudo su - +cd /apps/hbv3/wrf-openmpi/WRF-4.1.5/run +cp -f namelist.input namelist.input.old +cp -f /data/wrfdata/WRF_benchmarks/cases/new_conus2.5km/namelist.input . +cp /apps/hbv3/wps-openmpi/WPS-4.1/met_em.d0*.nc . +mpirun --allow-run-as-root ./real.exe ``` -git clone https://github.com/akirakyle/WRF_benchmarks.git + +Expected results: +The following files should be generated in /apps/hbv3/wrf-openmpi/WRF-4.1.5/run/ +- wrfbdy_d01 +- wrfinput_d01 +``` +ls -l /apps/hbv3/wrf-openmpi/WRF-4.1.5/run/*_d0* +``` + +#### Change Permissions on Files +Grant permission to users that will run the WRF to access the files on /data and /apps. +For my environment, I used "azureadmin" as CycleCloud admin. +Change the user below accordingly. + +``` +# grant permission to users that will run the WRF to access the files. +chown -R :cyclecloud /data +chown -R :cyclecloud /apps +chmod -R g+w /data +chmod -R g+w /apps +``` + +**If you get here, you have completed the WRF v4 setup!** + +Now you can shutdown and terminate the worker node (HBv3) used to perform these setup procedures. + +## Running and Testing + +Connect to head node of your cluster and submit WRF v4 simulation job: + +> Where SKU_TYPE is the sku type you are running on and INPUTDIR contains the location of wrf input files (namelist.input, wrfbdy_d01 and wrfinput_d01) + +- Test 1 + - SKU: Standard_HB120rs_v2 + - Nodes: 1 + - Processes per node: 60 + - MPI processes per node: 30 ``` +mkdir ~/test1 +cd ~/test1 -Download the raw data from https://rda.ucar.edu/datasets/ds084.1/ -Go to the "data Access" section and download the following data (2018, 2018-06-17, gfs.0p25.2018061700.f000.grib2, gfs.0p25.2018061712.f384.grib2) +qsub -l select=1:nodearray=execute1:ncpus=60:mpiprocs=30,place=scatter:excl -v "SKU_TYPE=hbv3,INPUTDIR=/apps/hbv3/wrf-openmpi/WRF-4.1.5/run" /data/azurehpc/apps/wrf/run_wrf_openmpi.pbs +``` -Modify your namelist.wps file, setting the correct paths for geog_data_path, opt_geogrid_tbl_path and opt_metgrid_tbl_path. +- Test 2 + - SKU: Standard_HB120rs_v2 + - Nodes: **2** + - Processes per node: 60 + - MPI processes per node: 30 +``` +mkdir ~/test2 +cd ~/test2 -cd to your WPS installation directory and copy the new_conus2.5km namelist.wps to this location. +qsub -l select=2:nodearray=execute1:ncpus=60:mpiprocs=30,place=scatter:excl -v "SKU_TYPE=hbv3,INPUTDIR=/apps/hbv3/wrf-openmpi/WRF-4.1.5/run" /data/azurehpc/apps/wrf/run_wrf_openmpi.pbs +``` -Download WPS v4 geopraphical static data for WPS v4. (https://www2.mmm.ucar.edu/wrf/users/download/get_sources_wps_geog.html), Download topo_gmted2010_30s, modis_landuse_20class_30s_with_lakes, soiltemp_1deg, soiltype_top_30s, albedo_modis, greenfrac_fpar_modis, lai_modis_10m, maxsnowalb_modis_10m and orogwd_10m. Extract into a directory. +- Test 3 + - SKU: Standard_HB120rs_v2 + - Nodes: **3** + - Processes per node: 60 + - MPI processes per node: 30 ``` -mpirun ./geogrid.exe +mkdir ~/test3 +cd ~/test3 + +qsub -l select=3:nodearray=execute1:ncpus=60:mpiprocs=30,place=scatter:excl -v "SKU_TYPE=hbv3,INPUTDIR=/apps/hbv3/wrf-openmpi/WRF-4.1.5/run" /data/azurehpc/apps/wrf/run_wrf_openmpi.pbs ``` -The file geo_em.d01.nc should be created. + +- Test 4 + - SKU: Standard_HB120rs_v2 + - Nodes: 3 + - Processes per node: 60 + - MPI processes per node: **60** ``` -ln -s ungrib/Variable_Tables/Vtable.GFS Vtable +mkdir ~/test4 +cd ~/test4 + +qsub -l select=3:nodearray=execute1:ncpus=60:mpiprocs=60,place=scatter:excl -v "SKU_TYPE=hbv3,INPUTDIR=/apps/hbv3/wrf-openmpi/WRF-4.1.5/run" /data/azurehpc/apps/wrf/run_wrf_openmpi.pbs ``` + +- Test 5 + - SKU: **Standard_HB120rs_v3** + - Nodes: 3 + - Processes per node: 60 + - MPI processes per node: 60 ``` -./link_grib.csh /location/of/NCEP_GFS_Model_Run_data/gfs.0p25.20180617* +mkdir ~/test5 +cd ~/test5 + +qsub -l select=3:nodearray=execute1:ncpus=60:mpiprocs=60,place=scatter:excl -v "SKU_TYPE=hbv3,INPUTDIR=/apps/hbv3/wrf-openmpi/WRF-4.1.5/run" /data/azurehpc/apps/wrf/run_wrf_openmpi.pbs ``` + +- Test 6 + - SKU: **Standard_HB120rs_v2** + - Nodes: **4** + - Processes per node: 60 + - MPI processes per node: 60 ``` -./ungrib.exe >& ungrib.log +mkdir ~/test6 +cd ~/test6 + +qsub -l select=4:nodearray=execute1:ncpus=60:mpiprocs=60,place=scatter:excl -v "SKU_TYPE=hbv3,INPUTDIR=/apps/hbv3/wrf-openmpi/WRF-4.1.5/run" /data/azurehpc/apps/wrf/run_wrf_openmpi.pbs ``` + +- Test 7 + - SKU: **Standard_HB120-64rs_v2** + - Nodes: **3** + - Processes per node: **64** + - MPI processes per node: **64** ``` -mpirun ./metgrid.exe +mkdir ~/test7 +cd ~/test7 + +qsub -l select=3:nodearray=execute1:ncpus=64:mpiprocs=64,place=scatter:excl -v "SKU_TYPE=hbv3,INPUTDIR=/apps/hbv3/wrf-openmpi/WRF-4.1.5/run" /data/azurehpc/apps/wrf/run_wrf_openmpi.pbs ``` -Should see the following files + +- Test 8 + - SKU: **Standard_HB120-64rs_v3** + - Nodes: 3 + - Processes per node: 64 + - MPI processes per node: 64 ``` --rw-rw-r--. 1 hpcuser hpcuser 79075286 Dec 12 04:11 met_em.d01.2018-06-17_12:00:00.nc --rw-rw-r--. 1 hpcuser hpcuser 78622472 Dec 12 04:11 met_em.d01.2018-06-17_09:00:00.nc --rw-rw-r--. 1 hpcuser hpcuser 79054700 Dec 12 04:10 met_em.d01.2018-06-17_06:00:00.nc --rw-rw-r--. 1 hpcuser hpcuser 78668607 Dec 12 04:10 met_em.d01.2018-06-17_03:00:00.nc --rw-rw-r--. 1 hpcuser hpcuser 79435709 Dec 12 04:10 met_em.d01.2018-06-17_00:00:00.nc +mkdir ~/test8 +cd ~/test8 + +qsub -l select=3:nodearray=execute1:ncpus=64:mpiprocs=64,place=scatter:excl -v "SKU_TYPE=hbv3,INPUTDIR=/apps/hbv3/wrf-openmpi/WRF-4.1.5/run" /data/azurehpc/apps/wrf/run_wrf_openmpi.pbs ``` -cd WRF v4 run directory -cp new_conus2.5km namelist.input to this location -cp the file met_em*.nc files to this location +- Test 9 + - SKU: **Standard_HB120rs_v3** + - Nodes: 3 + - Processes per node: 64 + - MPI processes per node: 64 ``` -mpirun ./real.exe +mkdir ~/test9 +cd ~/test9 + +qsub -l select=3:nodearray=execute1:ncpus=64:mpiprocs=64,place=scatter:excl -v "SKU_TYPE=hbv3,INPUTDIR=/apps/hbv3/wrf-openmpi/WRF-4.1.5/run" /data/azurehpc/apps/wrf/run_wrf_openmpi.pbs ``` -The following files should be generated (These are the input files required to run WRF v4) + +- Test 10 + - SKU: Standard_HB120rs_v3 + - Nodes: 3 + - Processes per node: **120** + - MPI processes per node: **120** ``` --rw-rw-r--. 1 hpcuser hpcuser 31285895 Dec 12 15:50 wrfbdy_d01 --rw-rw-r--. 1 hpcuser hpcuser 127618030 Dec 12 15:50 wrfinput_d01 +mkdir ~/test10 +cd ~/test10 + +qsub -l select=3:nodearray=execute1:ncpus=120:mpiprocs=120,place=scatter:excl -v "SKU_TYPE=hbv3,INPUTDIR=/apps/hbv3/wrf-openmpi/WRF-4.1.5/run" /data/azurehpc/apps/wrf/run_wrf_openmpi.pbs ``` + +### Test Results +In the graph below you can compare the execution time from tests performed, with different number of nodes, cores, mpicores and SKUs: +![Import-Template1](images/wrf-test-results.png) + + +### Validate Job Status +To validate the status of the job submission, you can use: +``` +# check jobs history +qstat -x + +# check jobs history showing the wall clock duration +qstat -xG + +# check jobs history showing the execution details +qstat -xf +``` + + + + + + diff --git a/apps/wrf/run_real_openmpi.pbs b/apps/wrf/run_real_openmpi.pbs new file mode 100644 index 000000000..d735bc539 --- /dev/null +++ b/apps/wrf/run_real_openmpi.pbs @@ -0,0 +1,50 @@ +#!/bin/bash +SKU_TYPE=${1:-$SKU_TYPE} +SKU_TYPE=${SKU_TYPE:-"hbv2"} +INPUTDIR=${2:-$INPUTDIR} +INPUTDIR=${INPUTDIR:-"/apps/hbv2/wrf-openmpi/WRF-4.1.5/run"} + +echo "INPUTDIR:" $INPUTDIR +echo "SKU_TYPE:" $SKU_TYPE + +SHARED_APP=${SHARED_APP:-/apps} + +if [ -z $INPUTDIR ]; then + echo "INPUTDIR parameter is required" + exit 1 +fi + +if ! rpm -q python3; then + sudo yum install -y python3 +fi + +echo "source envs" +source /data/azurehpc/apps/wrf/env-variables + +NPROCS=`cat $PBS_NODEFILE | wc -l` +echo "NPROCS:"$NPROCS + +echo "PBS_O_WORKDIR:" $PBS_O_WORKDIR +echo "INPUTDIR: " ${INPUTDIR} + +cd ${INPUTDIR} +pwd + +mpi_options="-x LD_LIBRARY_PATH " +#if [ -n $LD_PRELOAD ]; then +# mpi_options+="-x LD_PRELOAD" +#fi +echo "PBS_NODEFILE:" $PBS_NODEFILE +echo "mpi_options: " $mpi_options + +#cp -f namelist.input namelist.input.old +#cp -f /data/wrfdata/WRF_benchmarks/cases/new_conus2.5km/namelist.input . +cp /apps/hbv2/wps-openmpi/WPS-4.1/met_em.d0*.nc . + +#### Run real.exe +echo "-- Run real.exe" +mpirun $mpi_options -n $NPROCS --hostfile $PBS_NODEFILE --bind-to numa ./real.exe +#mpirun --allow-run-as-root ./real.exe + + + diff --git a/apps/wrf/run_wps_openmpi.pbs b/apps/wrf/run_wps_openmpi.pbs new file mode 100644 index 000000000..0d7f7cf14 --- /dev/null +++ b/apps/wrf/run_wps_openmpi.pbs @@ -0,0 +1,60 @@ +#!/bin/bash +SKU_TYPE=${1:-$SKU_TYPE} +SKU_TYPE=${SKU_TYPE:-"hbv2"} +INPUTDIR=${2:-$INPUTDIR} +INPUTDIR=${INPUTDIR:-"/apps/hbv2/wps-openmpi/WPS-4.1/"} + +echo "INPUTDIR:" $INPUTDIR +echo "SKU_TYPE:" $SKU_TYPE + +SHARED_APP=${SHARED_APP:-/apps} + +if [ -z $INPUTDIR ]; then + echo "INPUTDIR parameter is required" + exit 1 +fi + +if ! rpm -q python3; then + sudo yum install -y python3 +fi + +echo "source envs" +source /data/azurehpc/apps/wrf/env-variables + +NPROCS=`cat $PBS_NODEFILE | wc -l` +echo "NPROCS:"$NPROCS + +echo "PBS_O_WORKDIR:" $PBS_O_WORKDIR +echo "INPUTDIR: " ${INPUTDIR} + +cd ${INPUTDIR} +pwd + +mpi_options="-x LD_LIBRARY_PATH " +#if [ -n $LD_PRELOAD ]; then +# mpi_options+="-x LD_PRELOAD" +#fi +echo "PBS_NODEFILE:" $PBS_NODEFILE +echo "mpi_options: " $mpi_options + +#### Run geogrid.exe +echo "-- Run geogrid.exe" +#mpirun $mpi_options -n $NPROCS --hostfile $PBS_NODEFILE --bind-to numa ./geogrid.exe +mpirun -n $NPROCS ./geogrid.exe + +echo "-- VTable" +ln -s ungrib/Variable_Tables/Vtable.GFS Vtable + +echo "-- link_grib.csh" +./link_grib.csh /data/wrfdata/gfs_files/gfs.0p25.* + +echo "-- ungrib.exe" +./ungrib.exe >& ungrib.log + +#### Run metgrid.exe +echo "-- Run metgrid.exe" +#mpirun $mpi_options -n $NPROCS --hostfile $PBS_NODEFILE --bind-to numa metgrid.exe +mpirun -n $NPROCS ./metgrid.exe + + + diff --git a/apps/wrf/run_wrf_openmpi.pbs b/apps/wrf/run_wrf_openmpi.pbs index 4e3f6a0d1..5e70b6969 100755 --- a/apps/wrf/run_wrf_openmpi.pbs +++ b/apps/wrf/run_wrf_openmpi.pbs @@ -1,7 +1,12 @@ #!/bin/bash SKU_TYPE=${1:-$SKU_TYPE} -SKU_TYPE=${SKU_TYPE:-hbv2} -INPUTDIR=$2 +SKU_TYPE=${SKU_TYPE:-"hbv2"} +INPUTDIR=${2:-$INPUTDIR} +INPUTDIR=${INPUTDIR:-"/apps/hbv2/wrf-openmpi/WRF-4.1.5/run"} + +echo "INPUTDIR:" $INPUTDIR +echo "SKU_TYPE:" $SKU_TYPE + SHARED_APP=${SHARED_APP:-/apps} if [ -z $INPUTDIR ]; then @@ -10,29 +15,18 @@ if [ -z $INPUTDIR ]; then fi if ! rpm -q python3; then - sudo yum install -y python3 + sudo yum install -y python3 fi -source /etc/profile.d/modules.sh -export MODULEPATH=${SHARED_APP}/modulefiles/${SKU_TYPE}:$MODULEPATH - -module use ${SHARED_APP}/modulefiles -module load spack/spack -source $SPACK_SETUP_ENV - -spack load netcdf-fortran^openmpi -spack load hdf5^openmpi -spack load perl -module use /usr/share/Modules/modulefiles - -module load mpi/openmpi-4.0.5 -module load gcc-9.2.0 -module load wrf/4.1.5-openmpi - -which mpicc -which wrf.exe +echo "source envs" +source /data/azurehpc/apps/wrf/env-variables NPROCS=`cat $PBS_NODEFILE | wc -l` +echo "NPROCS:"$NPROCS + +echo "PBS_O_WORKDIR:" $PBS_O_WORKDIR +echo "WRFROOT:" ${WRFROOT} +echo "INPUTDIR: " ${INPUTDIR} cd $PBS_O_WORKDIR ln -s ${WRFROOT}/run/* . @@ -43,5 +37,6 @@ mpi_options="-x LD_LIBRARY_PATH " if [ -n $LD_PRELOAD ]; then mpi_options+="-x LD_PRELOAD" fi +echo "PBS_NODEFILE:" $PBS_NODEFILE mpirun $mpi_options -n $NPROCS --hostfile $PBS_NODEFILE --bind-to numa wrf.exe diff --git a/apps/wrf/run_wrf_openmpi.slurm b/apps/wrf/run_wrf_openmpi.slurm new file mode 100644 index 000000000..e9b671a1d --- /dev/null +++ b/apps/wrf/run_wrf_openmpi.slurm @@ -0,0 +1,72 @@ +#!/bin/bash +#SBATCH --job-name=wrf +#SBATCH --partition=hpc +#SBATCH --ntasks-per-node=120 +#SBATCH --nodes=1 +#SBATCH --exclusive +#SBATCH --output=slurm-wrf-%j.out + +# Capture start time +start_time=$(date +%s) + +SKU_TYPE=${1:-$SKU_TYPE} +SKU_TYPE=${SKU_TYPE:-"hbv3"} +INPUTDIR=${2:-$INPUTDIR} +INPUTDIR=${INPUTDIR:-"/apps/hbv3/wrf-openmpi/WRF-4.1.5/run"} + +echo "INPUTDIR:" $INPUTDIR +echo "SKU_TYPE:" $SKU_TYPE + +SHARED_APP=${SHARED_APP:-/apps} + +if [ -z $INPUTDIR ]; then + echo "INPUTDIR parameter is required" + exit 1 +fi + +if ! rpm -q python3; then + sudo yum install -y python3 +fi + +echo "source envs" +source /data/azurehpc/apps/wrf/env-variables hbv3 + +echo "SLURM_SUBMIT_DIR:" $SLURM_SUBMIT_DIR +echo "WRFROOT:" ${WRFROOT} +echo "INPUTDIR: " ${INPUTDIR} + +cd $SLURM_SUBMIT_DIR +ln -s ${WRFROOT}/run/* . +cp ${INPUTDIR}/*_d01 . +cp ${INPUTDIR}/namelist.input . + +echo "LD_PRELOAD:" $LD_PRELOAD +mpi_options="-x LD_LIBRARY_PATH " +if [ -n $LD_PRELOAD ]; then + mpi_options+="-x LD_PRELOAD" +fi +echo "mpi_options:" $mpi_options + +echo "SLURM_JOB_NODELIST:" $SLURM_JOB_NODELIST +echo "SLURM_NTASKS:" $SLURM_NTASKS + +scontrol show hostname $SLURM_JOB_NODELIST > hostfile.txt +NPROCS=`cat hostfile.txt | wc -l` +echo "NPROCS:" $NPROCS + +echo "Running WRF.exe..." +mpirun $mpi_options -n $SLURM_NTASKS --hostfile hostfile.txt --bind-to numa wrf.exe + +# Capture end time +end_time=$(date +%s) +# Calculate duration +duration=$((end_time - start_time)) +# Format duration to hh:mm:ss +hours=$(printf "%02d" $((duration / 3600))) +minutes=$(printf "%02d" $(( (duration % 3600) / 60 ))) +seconds=$(printf "%02d" $((duration % 60))) +duration="${hours}:${minutes}:${seconds}" + +echo "WRF finished." +echo "Execution time: $duration seconds" + diff --git a/apps/wrf/start-full-run.sh b/apps/wrf/start-full-run.sh new file mode 100644 index 000000000..c9ec13d69 --- /dev/null +++ b/apps/wrf/start-full-run.sh @@ -0,0 +1,13 @@ +# Run WPS job +echo "Submiting WPS Job" +JOB_WPS=$(qsub -l select=1:nodearray=execute2:ncpus=120:mpiprocs=120,place=scatter:excl -v "SKU_TYPE=hbv2,INPUTDIR=/apps/hbv2/wps-openmpi/WPS-4.1/" /data/azurehpc/apps/wrf/run_wps_openmpi.pbs) +echo "WPS Job ID:$JOB_WPS" + +# Run Real job +echo "Submiting Real Job" +JOB_Real=$(qsub -W depend=afterok:$JOB_WPS -l select=1:nodearray=execute2:ncpus=120:mpiprocs=120,place=scatter:excl -v "SKU_TYPE=hbv2,INPUTDIR=/apps/hbv2/wrf-openmpi/WRF-4.1.5/run/" /data/azurehpc/apps/wrf/run_real_openmpi.pbs) +echo "Real Job ID:$JOB_Real" + +# Run WRF job +echo "Submiting WRF Job" +qsub -W depend=afterok:$JOB_Real -l select=1:nodearray=execute2:ncpus=120:mpiprocs=120,place=scatter:excl -v "SKU_TYPE=hbv2,INPUTDIR=/apps/hbv2/wrf-openmpi/WRF-4.1.5/run/" /data/azurehpc/apps/wrf/run_wrf_openmpi.pbs diff --git a/apps/wrf/submit-wrf-slurm.sh b/apps/wrf/submit-wrf-slurm.sh new file mode 100644 index 000000000..2662b2d8b --- /dev/null +++ b/apps/wrf/submit-wrf-slurm.sh @@ -0,0 +1,2 @@ +#!/bin/bash +sbatch --export=ALL,SKU_TYPE=hbv3,INPUTDIR=/apps/hbv3/wrf-openmpi/WRF-4.1.5/run /data/azurehpc/apps/wrf/run_wrf_openmpi.slurm diff --git a/experimental/wrf_on_cyclecloud/images/Create-WRF-Cluster1.png b/experimental/wrf_on_cyclecloud/images/Create-WRF-Cluster1.png new file mode 100644 index 000000000..c89ce02c8 Binary files /dev/null and b/experimental/wrf_on_cyclecloud/images/Create-WRF-Cluster1.png differ diff --git a/experimental/wrf_on_cyclecloud/images/Create-WRF-Cluster2.png b/experimental/wrf_on_cyclecloud/images/Create-WRF-Cluster2.png new file mode 100644 index 000000000..a0b6a4a1e Binary files /dev/null and b/experimental/wrf_on_cyclecloud/images/Create-WRF-Cluster2.png differ diff --git a/experimental/wrf_on_cyclecloud/images/Create-WRF-Cluster2a.png b/experimental/wrf_on_cyclecloud/images/Create-WRF-Cluster2a.png new file mode 100644 index 000000000..9f66c3ac5 Binary files /dev/null and b/experimental/wrf_on_cyclecloud/images/Create-WRF-Cluster2a.png differ diff --git a/experimental/wrf_on_cyclecloud/images/Create-WRF-Cluster3.png b/experimental/wrf_on_cyclecloud/images/Create-WRF-Cluster3.png new file mode 100644 index 000000000..ffde26db6 Binary files /dev/null and b/experimental/wrf_on_cyclecloud/images/Create-WRF-Cluster3.png differ diff --git a/experimental/wrf_on_cyclecloud/images/Import-Template1.png b/experimental/wrf_on_cyclecloud/images/Import-Template1.png new file mode 100644 index 000000000..74625254c Binary files /dev/null and b/experimental/wrf_on_cyclecloud/images/Import-Template1.png differ diff --git a/experimental/wrf_on_cyclecloud/images/NFS-Cluster1.png b/experimental/wrf_on_cyclecloud/images/NFS-Cluster1.png new file mode 100644 index 000000000..5d81fb0ec Binary files /dev/null and b/experimental/wrf_on_cyclecloud/images/NFS-Cluster1.png differ diff --git a/experimental/wrf_on_cyclecloud/images/NFS-Cluster2.png b/experimental/wrf_on_cyclecloud/images/NFS-Cluster2.png new file mode 100644 index 000000000..fd8fba9ce Binary files /dev/null and b/experimental/wrf_on_cyclecloud/images/NFS-Cluster2.png differ diff --git a/experimental/wrf_on_cyclecloud/images/NFS-Cluster3.png b/experimental/wrf_on_cyclecloud/images/NFS-Cluster3.png new file mode 100644 index 000000000..936367096 Binary files /dev/null and b/experimental/wrf_on_cyclecloud/images/NFS-Cluster3.png differ diff --git a/experimental/wrf_on_cyclecloud/images/Start-WRF-Cluster1.png b/experimental/wrf_on_cyclecloud/images/Start-WRF-Cluster1.png new file mode 100644 index 000000000..660a3cbe1 Binary files /dev/null and b/experimental/wrf_on_cyclecloud/images/Start-WRF-Cluster1.png differ diff --git a/experimental/wrf_on_cyclecloud/images/Start-WRF-Cluster2.png b/experimental/wrf_on_cyclecloud/images/Start-WRF-Cluster2.png new file mode 100644 index 000000000..162b362ba Binary files /dev/null and b/experimental/wrf_on_cyclecloud/images/Start-WRF-Cluster2.png differ diff --git a/experimental/wrf_on_cyclecloud/readme.md b/experimental/wrf_on_cyclecloud/readme.md new file mode 100644 index 000000000..8fe59f5b3 --- /dev/null +++ b/experimental/wrf_on_cyclecloud/readme.md @@ -0,0 +1,106 @@ +# Install and run WRF v4 and WPS v4 - End-to-end Setup guide for a Lab environment using Cycle Cloud + +Summary of this procedure: +- Installs CycleCloud Lab environment from scratch +- Creates NFS storage server using CycleCloud cluster template +- Installs WPS/WRF v4 software (via “azurehpc” scripts) + +## Prerequisites +- As this procedure uses HBv2 VMs to run WRFv4 simulations, you may need to request quota increase for this type of SKU in the subscription and region you will deploy the environment. + + +## Install Azure CycleCloud Lab environment +Follow the steps to [Install and Setup CycleCloud Lab Environment](../../tutorials/cyclecloud/install-cyclecloud.md) from scratch. + + +## Create NFS Storage cluster +- It can be possible to include an external NFS share at this point (in the example, I have shared from an NFS cluster using on CycleCloud template) +- As this procedure is for a lab environment we are creating a NFS storage cluster, using the standard template in Cycle Cloud. For production environments, you could use a high performance storage solutions, such as Azure NetApp Files (ANF), for instance. + + +![NFS-Cluster1](images/NFS-Cluster1.png) +![NFS-Cluster2](images/NFS-Cluster2.png) +![NFS-Cluster3](images/NFS-Cluster3.png) + +Changes: +- Change OS to use CentOS 7 versions +- Use +300GB storage size (space to download WRF data) +- Change cloud-init as following: +- Confirm the IP address of your NFS storage and change it below accordingly: + +``` +#!/bin/bash + +set -x +yum install -y epel-release +yum install -y Lmod at +systemctl enable --now atd.service +cat </mnt/exportfs.sh +#!/bin/bash +set -x +mkdir -p /mnt/exports/data /mnt/exports/apps +sudo exportfs -o rw,sync,no_root_squash 10.4.0.0/20:/mnt/exports/data +sudo exportfs -o rw,sync,no_root_squash 10.4.0.0/20:/mnt/exports/apps +EOF +chmod 755 /mnt/exportfs.sh +at now + 2 minute -f /mnt/exportfs.sh +``` + +Connect to NFS storage cluster and check the mounts are correct: +``` +# check mount +sudo exportfs -s +``` + +## Setup WRF cluster using HBv2 VM +Summary of steps: +- Import WRF cluster template (from “azurehpc” scripts) +- Start WRF cluster using HBv2 VM +- Install WPS/WRF v4 software (from “azurehpc” scripts) + + +### Import custom CycleCloud template for WRF + +- Start NFS storage cluster + +- SSH to it and download azurehpc GitHub repository to the /data folder. +``` +## Download azurehpc GitHub repository +cd /data +git clone https://github.com/Azure/azurehpc.git +``` + +Follow the procedures [here](https://docs.microsoft.com/en-us/azure/cyclecloud/tutorials/modify-cluster-template?view=cyclecloud-8#import-the-new-cluster-template) to upload the Cycle Cloud custom template created for WRF. +Use the template: [opbswrf-template.txt](opbswrf-template.txt) +``` +## Import CycleCloud template +cd /data/azurehpc/apps/wrf/ +cyclecloud import_template opbswrf -f opbswrf-template.txt --force +``` +After you import the template, you will see the WRF template in CycleCloud Portal: + +![Import-Template1](images/Import-Template1.png) + +### Create new WRF cluster +Choose the WRF Cluster name: +![Create-WRF-Cluster1](images/Create-WRF-Cluster1.png) + +Choose the SKUs you want use for testing and the subnet for the compute VMs: +![Create-WRF-Cluster2](images/Create-WRF-Cluster2a.png) + +Check **Additional NFS Mount** options and change to the correct NFS IP address, related to your environment. Don’t need to change NFS Mount Point and NFS Export Path +![Create-WRF-Cluster3](images/Create-WRF-Cluster3.png) + +Keep the default value for the other parameters, save it and start the cluster. + +### Spin Execute Node with HBv2 +After the cluster is up and running, start a worker node using HBv2 VM: +![Start-WRF-Cluster1](images/Start-WRF-Cluster1.png) +![Start-WRF-Cluster2](images/Start-WRF-Cluster2.png) +Click Add. + +### Install WPS/WRF v4 software (via “azurehpc” scripts) +Now you have a WRF cluster properly configured and running in Cycle Cloud. + +You can follow the instructions here to finish the WPS/WRF installation: [Install and run WPS and WRF v4 - Setup guide](/apps/wrf/readme.md) + diff --git a/tutorials/cyclecloud/images/DNS-Label1.jpg b/tutorials/cyclecloud/images/DNS-Label1.jpg new file mode 100644 index 000000000..37c76a90c Binary files /dev/null and b/tutorials/cyclecloud/images/DNS-Label1.jpg differ diff --git a/tutorials/cyclecloud/images/DNS-Label2.jpg b/tutorials/cyclecloud/images/DNS-Label2.jpg new file mode 100644 index 000000000..4bba7386c Binary files /dev/null and b/tutorials/cyclecloud/images/DNS-Label2.jpg differ diff --git a/tutorials/cyclecloud/images/Managed-Identity1.jpg b/tutorials/cyclecloud/images/Managed-Identity1.jpg new file mode 100644 index 000000000..0f967c354 Binary files /dev/null and b/tutorials/cyclecloud/images/Managed-Identity1.jpg differ diff --git a/tutorials/cyclecloud/install-cyclecloud.md b/tutorials/cyclecloud/install-cyclecloud.md new file mode 100644 index 000000000..51129e2e3 --- /dev/null +++ b/tutorials/cyclecloud/install-cyclecloud.md @@ -0,0 +1,105 @@ +# Install and Setup CycleCloud for a Lab environment + +Before you start, define which **Subscription**, **Resource Group**, **Region** and **Virtual Network** you will use to deploy your CycleCloud environment. + +## 1. Create VNET +Here we are creating a new Virtual Network for the CycleCloud environment. For this basic setup, we are using only CycleCloud and Compute subnets. But, you can create all subnets in this script, for future advanced scenarios. +You can use the default variable values below or change it, if you want to. +``` +################################################## +# Variables +################################################## +Location="southcentralus" +RG_VNET="rg-wrf-poc-$Location" + +# VNET HPC +VNET_NAME="vnet-wrf-poc-$Location" +SUBNET_NAME_CYCLECLOUD="cyclecloud-subnet" +SUBNET_NAME_USER="user-subnet" +SUBNET_NAME_COMPUTE="compute-subnet" +SUBNET_NAME_ANF="anf-subnet" +SUBNET_NAME_VISUALIZATION="visualization-subnet" + +# IP ranges +VNET_NAME_CIDR="10.4.0.0/20" +SUBNET_NAME_CYCLECLOUD_CIDR="10.4.0.0/28" +SUBNET_NAME_USER_CIDR="10.4.2.0/24" +SUBNET_NAME_COMPUTE_CIDR="10.4.4.0/22" +SUBNET_NAME_VISUALIZATION_CIDR="10.4.1.0/25" +SUBNET_NAME_ANF_CIDR="10.4.0.16/28" +################################################## + +# Create resource Group +az group create --name $RG_VNET --location $Location + +# Create VNET +az network vnet create --resource-group $RG_VNET --name $VNET_NAME --location $Location \ + --address-prefixes $VNET_NAME_CIDR + +# Create Subnets +az network vnet subnet create --address-prefix $SUBNET_NAME_CYCLECLOUD_CIDR --name $SUBNET_NAME_CYCLECLOUD \ + --resource-group $RG_VNET --vnet-name $VNET_NAME + +az network vnet subnet create --address-prefix $SUBNET_NAME_USER_CIDR --name $SUBNET_NAME_USER \ + --resource-group $RG_VNET --vnet-name $VNET_NAME + +az network vnet subnet create --address-prefix $SUBNET_NAME_COMPUTE_CIDR --name $SUBNET_NAME_COMPUTE \ + --resource-group $RG_VNET --vnet-name $VNET_NAME + +az network vnet subnet create --address-prefix $SUBNET_NAME_ANF_CIDR --name $SUBNET_NAME_ANF \ + --resource-group $RG_VNET --vnet-name $VNET_NAME --delegations "Microsoft.NetApp/volumes" + +az network vnet subnet create --address-prefix $SUBNET_NAME_VISUALIZATION_CIDR --name $SUBNET_NAME_VISUALIZATION \ + --resource-group $RG_VNET --vnet-name $VNET_NAME +``` + +## 2. Create CycleCloud from Azure Portal + +CycleCloud can be installed via Azure Marketplace, see [Quickstart - Install via Marketplace - Azure CycleCloud | Microsoft Docs](https://learn.microsoft.com/en-us/azure/cyclecloud/qs-install-marketplace?view=cyclecloud-8): +1. Choose your subscription from the Subscription dropdown +2. Select or create a new Resource Group that your CycleCloud instance will run in. +3. Name your CycleCloud instance using Virtual Machine name +4. Select the Region +5. Create the Username that you will use to log into the instance +6. Add or create your SSH public key +7. If you are planning on using [Managed Identities](https://learn.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/overview) (recommended), Select the Management tab and enable System assigned managed identity. +8. Click on the Review button and then the Create button + +## 3. Assign Contributor role on the subscription to managed-identity +From Azure Portal, go to Subscriptions and assign Contributor role to the CycleCloud VM managed Identity (Subscription > Access control (IAM) > Add > Add role assignment). +>You need to have **Contributor** permission on the Subscription. If you need more granular permissions, see: [Create a custom role and managed identity for CycleCloud](https://learn.microsoft.com/en-us/azure/cyclecloud/how-to/managed-identities?view=cyclecloud-8#create-a-custom-role-and-managed-identity-for-cyclecloud) + +## 4. Configure DNS name label for public IP (optional) +You can define a DNS Name for the CycleCloud VM public IP, it can facilitate the access to the VM. +Go to CycleCloud VM and click in DNS Name: +![DNS-Label1](images/DNS-Label1.jpg) + +Add a DNS Name and save: +![DNS-Label2](images/DNS-Label2.jpg) + +In case you want to setup CycleCloud in environments with limited internet access, check this article: [Operating in a locked down network] (https://learn.microsoft.com/en-us/azure/cyclecloud/how-to/running-in-locked-down-network?view=cyclecloud-8). + +## 5. Create a storage account +Cyclecloud requires a storage account for [locker access](https://learn.microsoft.com/en-us/azure/cyclecloud/how-to/storage-blobs?view=cyclecloud-8). +Create storage account from the Azure Portal: +- Select the Subscription, Resource Group, Region you defined previously. +- Define a **Name**, set **Performance** as Standard, **Redundancy** as LRS. +- Keep all other settings with default values and create it. + +## 6. Setup CycleCloud Portal +- Open the browser and access the CycleCloud Portal using the DNS Label created above or the public IP of the CycleCloud VM. +- Follow the steps described [here](https://learn.microsoft.com/en-us/azure/cyclecloud/qs-install-marketplace?view=cyclecloud-8#log-into-the-cyclecloud-application-server) + +## 7. Install CycleCloud CLI +- Follow these steps to [Install CycleCloud CLI](https://learn.microsoft.com/en-us/azure/cyclecloud/how-to/install-cyclecloud-cli?view=cyclecloud-8): +``` +wget https:///static/tools/cyclecloud-cli.zip --no-check-certificate +unzip cyclecloud-cli.zip +cd cyclecloud-cli-installer +./install.sh +``` +- Initialize CycleCloud CLI + 1. Run **cyclecloud initialize**. You will be prompted for the CycleServer URL, which is the FQDN of your application server. Enter it in the format https://FQDN. + 2. The installed Azure CycleCloud server uses either a Let's Encrypt SSL certificate, or a self-signed certificate. Type yes when asked to allow the certificate. + 3. When prompted username and password, enter the same username and password used in the CycleCloud Portal web interface. + 4. Run **cyclecloud show_cluster** to test CycleCloud CLI is working.