From 89138513bf78524b8bbc551a4ddb009ae8c07be7 Mon Sep 17 00:00:00 2001
From: Christoph Knote <christoph.knote@med.uni-augsburg.de>
Date: Fri, 21 Oct 2022 15:52:26 +0200
Subject: [PATCH] Simplify MPI

---
 blueprints/test_chemistry/config.bash              |  6 +-----
 machine_profiles/alcc_spack/batch_preambles/main   |  3 +--
 machine_profiles/alcc_spack/batch_preambles/spinup |  3 +--
 machine_profiles/alcc_spack/profile.bash           | 12 ++----------
 master.bash                                        |  5 +----
 5 files changed, 6 insertions(+), 23 deletions(-)

diff --git a/blueprints/test_chemistry/config.bash b/blueprints/test_chemistry/config.bash
index 44ec363..618c667 100644
--- a/blueprints/test_chemistry/config.bash
+++ b/blueprints/test_chemistry/config.bash
@@ -100,8 +100,4 @@ TUVDataPath="/alcc/gpfs2/home/mbees/data/tuv/TUV.phot.bz2"
 
 # --- overriding default values ---
 
-nNodesMain=1
-nTasksPerNodeMain=9
-
-nNodesSpinup=1
-nTasksPerNodeSpinup=9
+nTasks=9
diff --git a/machine_profiles/alcc_spack/batch_preambles/main b/machine_profiles/alcc_spack/batch_preambles/main
index 51d7947..b0189e7 100644
--- a/machine_profiles/alcc_spack/batch_preambles/main
+++ b/machine_profiles/alcc_spack/batch_preambles/main
@@ -3,8 +3,7 @@
 #SBATCH -o __runDir__/__mainJobName__.%j.%N.out
 #SBATCH -D __runDir__
 #SBATCH -J __mainJobName__
-#SBATCH --nodes=__nNodesMain__
-#SBATCH --ntasks-per-node=__nTasksPerNodeMain__
+#SBATCH --ntasks=__nTasksMain__
 #SBATCH --mem-per-cpu=2200
 #SBATCH --mail-type=FAIL
 #SBATCH --mail-user=christoph.knote@med.uni-augsburg.de
diff --git a/machine_profiles/alcc_spack/batch_preambles/spinup b/machine_profiles/alcc_spack/batch_preambles/spinup
index 5693ae6..c449826 100644
--- a/machine_profiles/alcc_spack/batch_preambles/spinup
+++ b/machine_profiles/alcc_spack/batch_preambles/spinup
@@ -3,8 +3,7 @@
 #SBATCH -o __runDir__/__spinupJobName__.%j.%N.out
 #SBATCH -D __runDir__
 #SBATCH -J __spinupJobName__
-#SBATCH --nodes=__nNodesSpinup__
-#SBATCH --ntasks-per-node=__nTasksPerNodeSpinup__
+#SBATCH --ntasks=__nTask__
 #SBATCH --mem-per-cpu=2200
 #SBATCH --mail-type=FAIL
 #SBATCH --mail-user=christoph.knote@med.uni-augsburg.de
diff --git a/machine_profiles/alcc_spack/profile.bash b/machine_profiles/alcc_spack/profile.bash
index 5621441..99a8bac 100644
--- a/machine_profiles/alcc_spack/profile.bash
+++ b/machine_profiles/alcc_spack/profile.bash
@@ -4,21 +4,13 @@ module load wrf wrf-chem wps nco netcdf-c netcdf-fortran anthro_emis fire_emis m
 
 ulimit -s unlimited
 
-export OMPI_MCA_PML=ucx
-export UCX_TLS=tcp,sm
-export UCX_NET_DEVICES=eth5
-
 # --- MPI settings ---
 
-mpiCommandPre="mpirun"
+mpiCommandPre="mpirun -mca pml ucx -mca coll ^hcoll -x UCX_TLS=tcp,shm -x UCX_NET_DEVICES=eth5"
 mpiCommandMain="mpirun -mca pml ucx -mca coll ^hcoll -x UCX_TLS=tcp,shm -x UCX_NET_DEVICES=eth5"
 mpiCommandReal=${mpiCommandPre}
 
-nNodesMain=1
-nTasksPerNodeMain=64
-
-nNodesSpinup=1
-nTasksPerNodeSpinup=64
+nTasks=64
 
 # --- Batch system ---
 
diff --git a/master.bash b/master.bash
index 3fd791f..4a4a74c 100755
--- a/master.bash
+++ b/master.bash
@@ -267,10 +267,7 @@ s/__refDateTxt__/${dateTxt}/g
 s/__domains__/${max_dom}/g
 s/__removeRunDir__/${removeRunDir}/g
 s/__metIncSec__/${metIncSec}/g
-s/__nNodesMain__/${nNodesMain}/g
-s/__nTasksPerNodeMain__/${nTasksPerNodeMain}/g
-s/__nNodesSpinup__/${nNodesSpinup}/g
-s/__nTasksPerNodeSpinup__/${nTasksPerNodeSpinup}/g
+s/__nTasks__/${nTasks}/g
 EOF
 
 # real run from -spinup to +fcstTime
-- 
GitLab