diff --git a/blueprints/operational_chemistry/machine_specific/alcc_spack/batch_preambles/main b/blueprints/operational_chemistry/machine_specific/alcc_spack/batch_preambles/main new file mode 100644 index 0000000000000000000000000000000000000000..3371ef8518d10503420e3c066fd61d9bfa698f99 --- /dev/null +++ b/blueprints/operational_chemistry/machine_specific/alcc_spack/batch_preambles/main @@ -0,0 +1,11 @@ +#!/bin/bash -l +#SBATCH --partition=alcc1,epyc +#SBATCH -o __runDir__/__mainJobName__.%j.%N.out +#SBATCH -D __runDir__ +#SBATCH -J __mainJobName__ +#SBATCH --nodes=3 +#SBATCH --ntasks-per-node=28 +#SBATCH --mem-per-cpu=2000 +#SBATCH --mail-type=FAIL +#SBATCH --mail-user=christoph.knote@med.uni-augsburg.de +#SBATCH --time=18:00:00 diff --git a/blueprints/operational_chemistry/machine_specific/alcc_spack/batch_preambles/post b/blueprints/operational_chemistry/machine_specific/alcc_spack/batch_preambles/post new file mode 100644 index 0000000000000000000000000000000000000000..9cecb891d441d841608a66d6cc6bd2c874488598 --- /dev/null +++ b/blueprints/operational_chemistry/machine_specific/alcc_spack/batch_preambles/post @@ -0,0 +1,10 @@ +#!/bin/bash -l +#SBATCH --partition=alcc1 +#SBATCH -o __runDir__/__postJobName__.%j.%N.out +#SBATCH -D __runDir__ +#SBATCH -J __postJobName__ +#SBATCH --ntasks=1 +#SBATCH --mem=5G +#SBATCH --mail-type=FAIL +#SBATCH --mail-user=christoph.knote@med.uni-augsburg.de +#SBATCH --time=02:00:00 diff --git a/blueprints/operational_chemistry/machine_specific/alcc_spack/batch_preambles/pre b/blueprints/operational_chemistry/machine_specific/alcc_spack/batch_preambles/pre new file mode 100644 index 0000000000000000000000000000000000000000..2bb27db5fd1bfd8c733b1f17bd49b1e301a0c4b2 --- /dev/null +++ b/blueprints/operational_chemistry/machine_specific/alcc_spack/batch_preambles/pre @@ -0,0 +1,10 @@ +#!/bin/bash -l +#SBATCH --partition=alcc1 +#SBATCH -o __runDir__/__preJobName__.%j.%N.out +#SBATCH -D __runDir__ +#SBATCH -J __preJobName__ +#SBATCH --ntasks=1 +#SBATCH --mem=10G +#SBATCH --mail-type=FAIL +#SBATCH --mail-user=christoph.knote@med.uni-augsburg.de +#SBATCH --time=03:00:00 diff --git a/blueprints/operational_chemistry/machine_specific/alcc_spack/batch_preambles/spinup b/blueprints/operational_chemistry/machine_specific/alcc_spack/batch_preambles/spinup new file mode 100644 index 0000000000000000000000000000000000000000..8fe2f2f8997e47f846efb450c4a8b72190b59911 --- /dev/null +++ b/blueprints/operational_chemistry/machine_specific/alcc_spack/batch_preambles/spinup @@ -0,0 +1,11 @@ +#!/bin/bash -l +#SBATCH --partition=alcc1,epyc +#SBATCH -o __runDir__/__spinupJobName__.%j.%N.out +#SBATCH -D __runDir__ +#SBATCH -J __spinupJobName__ +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=28 +#SBATCH --mem-per-cpu=2000 +#SBATCH --mail-type=FAIL +#SBATCH --mail-user=christoph.knote@med.uni-augsburg.de +#SBATCH --time=01:00:00 diff --git a/blueprints/operational_chemistry/machine_specific/alcc_spack/batch_preambles/staging b/blueprints/operational_chemistry/machine_specific/alcc_spack/batch_preambles/staging new file mode 100644 index 0000000000000000000000000000000000000000..66b09ee6b48048058b626d7a6642151323cdc194 --- /dev/null +++ b/blueprints/operational_chemistry/machine_specific/alcc_spack/batch_preambles/staging @@ -0,0 +1,10 @@ +#!/bin/bash -l +#SBATCH --partition=alcc1 +#SBATCH -o __runDir__/__stagingJobName__.%j.%N.out +#SBATCH -D __runDir__ +#SBATCH -J __stagingJobName__ +#SBATCH --ntasks=1 +#SBATCH --mem=5G +#SBATCH --mail-type=FAIL +#SBATCH --mail-user=christoph.knote@med.uni-augsburg.de +#SBATCH --time=01:00:00 diff --git a/blueprints/operational_chemistry/machine_specific/alcc_spack/config.bash b/blueprints/operational_chemistry/machine_specific/alcc_spack/config.bash new file mode 100644 index 0000000000000000000000000000000000000000..c18860ec50020f854a9935c74372bb69ffcfadbb --- /dev/null +++ b/blueprints/operational_chemistry/machine_specific/alcc_spack/config.bash @@ -0,0 +1,116 @@ +#!/bin/bash +# ------------------------------------------------------------------------------ +# WRFOTRON v 2.0b +# Christoph Knote (LMU Munich, Germany) +# 06/2016 +# christoph.knote@lmu.de +# ------------------------------------------------------------------------------ + +# path to the WRFotron installation +chainDir=${HOME}/wrfotron + +# --- Executable locations --- + +# WPS installation directory +WPSDir=${WPS_SRC_PATH} +# WRF installation directory +WRFDir=${WRF_SRC_PATH} + +# --- Input data settings --- + +# path to geogrid input data +geogDir=/alcc/gpfs2/home/mbees/data/geog/ + +# meteo input +# Vtable for the chosen meteo input +metVtableFile=${WPS_SRC_PATH}/ungrib/Variable_Tables/Vtable.GFS +# time increment in hours +metInc=1 +# full path to a met input file - you can use any "%<>" abbreviations known +# to the "date" command +metFilePattern="/alcc/gpfs2/home/mbees/data/meteo/GFS/GF%Y%m%d%H" +# example: +# "/glade/p/rda/data/ds083.2/grib2/%Y/%Y.%m/fnl_%Y%m%d_%H_00.grib2" + +# --- Pre/Postprocessing settings --- + +# prepararation script +preScriptPath=NONEXISTENT.bash + +# postprocessing scripts (arbitrary) +postScriptPath=NONEXISTENT.bash +# postprocessing scripts (actions for each wrfout file) +postPerFileScriptPath=NONEXISTENT.bash + +# --- Working directories --- + +# where the WRF will be run (some fast, large disk like "scratch" or similar) +workDir=${SCRATCH}/WRF/work/ + +# where the unprocessed WRF output will be stored +stagingRootDir=${SCRATCH}/WRF/staging/ +# where the WRF output will be stored +archiveRootDir=${SCRATCH}/archive/WRF/ +# where the WRF restart files will be stored +restartRootDir=${SCRATCH}/WRF/restart/ + +# remove run directory after run is finished? +removeRunDir=false + +# --- MPI settings --- + +mpiCommandPre="mpirun /usr/bin/time -v" +mpiCommandMain="mpirun /usr/bin/time -v" +mpiCommandReal=${mpiCommandPre} + +# --- Batch system --- + +# argument to submit a job in a held state +batchHoldArgument="--hold" +# command to release a held job +batchReleaseCommand="scontrol release" +# command to submit jobs to the queueing system +batchSubmitCommand=sbatch +# dependency argument for chaining runs upon submission +batchDepArgument="--dependency=afterok:__id__" +# sed command ("used as s/__command/\1/") to retrieve job run PID upon +# submission with $batchSubmitCommand +batchPidSedCommand="Submitted batch job \(.*\)" + +# --- Chemistry --- + +withChemistry=true + +# WRF-Chem installation directory +WRFChemDir=${WRF_CHEM_SRC_PATH} + +# path to utility executables +meganBioEmissBin=megan_bio_emiss +mozbcBin=mozbc +weselyBin=wesely +exo_coldensBin=exo_coldens +anthro_emisBin=anthro_emis +fire_emisBin=fire_emis + +# path to MEGAN input data +MEGANdir=/alcc/gpfs2/home/mbees/data/emissions/biogenic/MEGAN + +# use anthro_emiss or predefined files? +emissUseAnthroEmiss=false +# raw emission input - the files you read in with anthro_emiss +emissDir=/alcc/gpfs2/home/mbees/data/emissions/anthropogenic/EDGARv5/MOZART_MOSAIC +# emission conversion script for anthro_emis - must match emissions in emissDir +emissInpFile=emis_edgarv5_mozmos.inp +# year the emissions are valid for (for offset calculation) +emissYear=2015 + +# FINN fires +fireFilePattern="/alcc/gpfs2/home/mbees/data/emissions/fires/FINN/GLOB_MOZ4_%Y%j.txt" +fireInpFile=finn_fires.inp + +# boundary condition input +chembcFilePattern="/alcc/gpfs2/home/mbees/data/chembc/WACCM/WACCM%Y%m%d" +chembcInpFile=waccm.inp + +# TUV photolysis option 4 data file +TUVDataPath="/alcc/gpfs2/home/mbees/data/tuv/TUV.phot.bz2" diff --git a/jobs/pre/20_chem_recreate_megan.bash b/jobs/pre/20_chem_recreate_megan.bash index 5afdcbd0efc7aa0902c1697d29a85b63bb01fb19..0612f97f59bbfb258277afc2108577ce8da7478b 100644 --- a/jobs/pre/20_chem_recreate_megan.bash +++ b/jobs/pre/20_chem_recreate_megan.bash @@ -20,8 +20,7 @@ fi if [ ! -f ${workDir}/${experiment}/wrfbiochemi_d01 ] then echo "MEGAN input files do not not exist - recreating from scratch. Grab a coffee..." - ln -s ${WRFMEGANdir}/megan_bio_emiss . - ./megan_bio_emiss < megan_bio_emiss.inp > megan_bio_emiss.out + ${meganBioEmissBin} < megan_bio_emiss.inp > megan_bio_emiss.out for domain in $(seq -f "0%g" 1 ${max_dom}) do meganDataFile=${workDir}/${experiment}/wrfbiochemi_d${domain} diff --git a/jobs/pre/30_chem_wes_coldens.bash b/jobs/pre/30_chem_wes_coldens.bash index a9aaeddcc3b59055f2cd0294fa0b1296f6b612e1..86e525b853603b7eb0ea26f7bcd4cc0d5df555e3 100644 --- a/jobs/pre/30_chem_wes_coldens.bash +++ b/jobs/pre/30_chem_wes_coldens.bash @@ -2,8 +2,6 @@ # Wesely, Exo_coldens # ----------------------------------------------------------------------------- -ln -s ${WRFmztoolsdir}/* . +${weselyBin} < wesely.inp > wesely.out -./wesely < wesely.inp > wesely.out - -./exo_coldens < exo_coldens.inp > exo_coldens.out +${exo_coldensBin} < exo_coldens.inp > exo_coldens.out diff --git a/jobs/pre/50_chem_anthro_emiss.bash b/jobs/pre/50_chem_anthro_emiss.bash index fdd9059d47cdb7822a718ec34b339e5dba85b40f..e9eff6b1900f0c41184bf6f08e9679addedb202a 100644 --- a/jobs/pre/50_chem_anthro_emiss.bash +++ b/jobs/pre/50_chem_anthro_emiss.bash @@ -6,13 +6,11 @@ if [ "$emissUseAnthroEmiss" = "true" ] then if [ -f anthro_emis.inp ] then - # using anthro_emiss - ln -s ${WRFanthrodir}/anthro_emis . # difference in years for emission offset emissYearOffset=0 let "emissYearOffset = __startYear__ - emissYear" sed -e "s/__emissYearOffset__/${emissYearOffset}/g" -i anthro_emis.inp - ./anthro_emis < anthro_emis.inp > anthro_emis.out + ${anthro_emisBin} < anthro_emis.inp > anthro_emis.out fi else if [ -f anthro_emis.inp ] @@ -28,13 +26,11 @@ if [ -f anthro_emis.inp ] if [ ! -f ${workDir}/${experiment}/wrfchemi_d01 ] then - # using anthro_emiss - ln -s ${WRFanthrodir}/anthro_emis . # difference in years for emission offset emissYearOffset=0 let "emissYearOffset = __startYear__ - emissYear" sed -e "s/__emissYearOffset__/${emissYearOffset}/g" -i anthro_emis.inp - ./anthro_emis < anthro_emis.inp > anthro_emis.out + ${anthro_emisBin} < anthro_emis.inp > anthro_emis.out for domain in $(seq -f "0%g" 1 ${max_dom}) do diff --git a/jobs/pre/60_chem_fire_emiss.bash b/jobs/pre/60_chem_fire_emiss.bash index 92463279e4ffba445c35886de631196aaa26218b..029bde45d58071c2a0d75a5aa510dabe1e8b1bf6 100644 --- a/jobs/pre/60_chem_fire_emiss.bash +++ b/jobs/pre/60_chem_fire_emiss.bash @@ -6,7 +6,6 @@ # using fire_emis: if [ -f fire_emis.inp ] then - ln -s ${WRFfiredir}/fire_emis . ln -s $(dirname ${fireFilePattern})/*nc . @@ -58,5 +57,5 @@ EOF fi /bin/sed -s "s/__domain__/${max_dom}/" -i fire_emis.inp - ./fire_emis < fire_emis.inp > fire_emis.out + ${fire_emisBin} < fire_emis.inp > fire_emis.out fi diff --git a/jobs/pre/70_chem_mozbc.bash b/jobs/pre/70_chem_mozbc.bash index 2a27c3a35548fbc969e40919e18f9d57614164f9..e3b141ad083b73ca4cb4a423851c09a212693cd9 100644 --- a/jobs/pre/70_chem_mozbc.bash +++ b/jobs/pre/70_chem_mozbc.bash @@ -4,7 +4,6 @@ if [ -f mozbc_outer.inp ] then -ln -s $WRFMOZARTdir/mozbc . let totDays="((spinupTime+fcstTime)/24)+2" i=0 @@ -24,11 +23,11 @@ do done # first domain, always -./mozbc < mozbc_outer.inp > mozbc.log +${mozbcBin} < mozbc_outer.inp > mozbc.log for domain in $(seq 2 ${max_dom}) do /bin/sed -s "s/__domain__/${domain}/" < mozbc_inner.inp > mozbc_inner_d${domain}.inp - ./mozbc < mozbc_inner_d${domain}.inp >> mozbc.log + ${mozbcBin} < mozbc_inner_d${domain}.inp >> mozbc.log done fi diff --git a/machine_profiles/alcc_spack b/machine_profiles/alcc_spack new file mode 100644 index 0000000000000000000000000000000000000000..55b749ffd484359386e5855520d0e25ea1b04e21 --- /dev/null +++ b/machine_profiles/alcc_spack @@ -0,0 +1,3 @@ +module load wrf wps anthro_emis fire_emis megan_bio_emiss mozbc wesely-exo_coldens + +ulimit -s unlimited