DINE

module purge
module load oneAPI_comp/2021.3.0
module load  gnu_comp/9.3.0 gsl
module load allinea/ddt/20.2.1

CXX=icpc ./configure --with-mpi=mpiicpc --with-multithreading=omp CXXFLAGS="-I/cosma/local/gsl/2.4/include -g -std=c++17 -fopenmp -march=native -O3 -funroll-loops -DnoMPISupportsSingleSidedCommunication" LDFLAGS=-fopenmp --enable-loadbalancing --enable-exahype --enable-particles --enable-blockstructured

python3 ccz4.py -impl fv-adaptive-enclave -s single-puncture -maxh 0.3 -ps 3 -plt 1.0 -et 5 -tn sbh_han_final -outdir . -interp linear --add-tracer 2 

sbatch -x b115,b116 --nodes 14 -t 4000 submit_var.sh 32 peano4_sbh_han_final

Submission script:

#!/bin/bash -l
#SBATCH -o %A.out
#SBATCH -e %A.err
#SBATCH -p bluefield1
#SBATCH -A durham
#SBATCH -t 4000
#SBATCH --ntasks-per-node=1
#SBATCH --nodes=14
#SBATCH -J CCZ4SBH
module purge
module load oneAPI_comp/2021.3.0
module load  gnu_comp/9.3.0 gsl
module load allinea/ddt/20.2.1
export UCX_TLS=ud,sm,self
OMP_NUM_THREADS=${1} mpirun   ./${2}  --timeout 10000
#OMP_NUM_THREADS=${1} ddt --connect mpirun   ./${2}  --timeout 10000

Submit as such:

First argument is number of threads, second argument is the executable

sbatch -x b115,b116 --nodes 14 -t 4300 submit_var.sh 32 peano4_sbh_han_final

5 Seconds data

library(tidyverse)
default <- c("n0", "n1", "t", "x","y","z")
vnames  <- sprintf("Q%s",seq(0:64))

read_csv("/home/gcgt96/Exa/Scaling/2020-08-24-DINE-SBH/5seconds/zzsbh_han_final-rank-7-snapshot-61.csv", col_names=c(default, vnames), skip=1) -> df

pr

commit 6eb2c881fbed9a44f39c1230136ca3b6d4d98d7d
Author: fin_zhang <dggx86@durham.ac.uk>
Date:   Mon Aug 23 14:51:02 2021 +0100

    gravity fixed


module load oneAPI_comp/2021.3.0 gsl gnu_comp/9.3.0


Currently Loaded Modulefiles:
 1) gsl/2.4(default)   2) python/3.6.5   3) ucx/1.10.1   4) oneAPI_comp/2021.3.0   5) gnu_comp/9.3.0
 
 

CXX=icpc ./configure --with-mpi=mpiicpc --with-multithreading=omp CXXFLAGS="-g -std=c++17 -fopenmp -march=native -O3 -funroll-loops -DnoMPISupportsSingleSidedCommunication" LDFLAGS=-fopenmp --enable-loadbalancing --enable-exahype -
-enable-particles --enable-blockstructured


python3 ccz4.py -impl fv-adaptive-enclave -s single-puncture -maxh 0.3 -ps 3 -plt 0.0 -et 0.1 -tn sbh03_lb00  --ReSwi 2 -outdir . -interp linear -lbq 0.0



#!/bin/bash -l
#SBATCH -o %A.out
#SBATCH -e %A.err
#SBATCH -p bluefield1
#SBATCH -A durham
#SBATCH -t 100
#SBATCH --ntasks-per-node=1
#SBATCH --nodes=14
#SBATCH -J testmpi
module purge
module load oneAPI_comp/2021.3.0
module load  gnu_comp/9.3.0 gsl
module load allinea/ddt/20.2.1
export UCX_TLS=ud,sm,self
OMP_NUM_THREADS=${1} mpirun  ./${2}  --timeout 10000

gnu

./configure --with-mpi=mpicxx --with-multithreading=omp CXXFLAGS="-I/cosma/local/dine/spack/opt/spack/linux-centos7-zen2/gcc-9.3.0/openmpi-4.1.1-puzre5jhl7hkkpgvkaz4ohuux
rgyc4ud/include -g -std=c++17 -fopenmp -march=native -O3 -funroll-loops -DnoMPISupportsSingleSidedCommunication" LDFLAGS="-fopenmp -L/cosma/local/dine/spack/opt/spack/linux-centos7-zen2/gcc-9.3.0/openmpi-4.1.1-puzre5jhl7hkkpgvkaz4ohuuxrgy
c4ud/lib -lmpi"