Experimentation

IntelOneAPI + MPI

export FLAVOUR_NOCONFLICT=1
module load gcc advisor oneapi intelmpi

./configure --with-multithreading=omp --enable-exahype --enable-loadbalancing --enable-blockstructured --with-mpi="I_MPI_CXX=icpx mpiicpc" CC=icx CXX=icpx CXXFLAGS="-std=c++17 -g -fiopenmp -funroll-loops -O3" LDFLAGS="-L/apps/developers/compilers/intel/2021.4/1/default/compiler/2021.4.0/linux/compiler/lib/intel64_lin -liomp5"

AOCC

module load aocc likwid


./configure CXX=clang++ CC=clang LDFLAGS="-fopenmp -flto -lstdc++fs -L/apps/developers/libraries/likwid/5.2.0/1/default -llikwid" CXXFLAGS="-Ofast -std=c++17 -fopenmp -march=native -mtune=native -funroll-loops -DLIKWID_PERFMON"  --with-multithreading=omp --enable-exahype --enable-loadbalancing --enable-blockstructured

#!/bin/bash
#SBATCH --job-name=regfusA
#SBATCH -p multi
#SBATCH -N 1
#SBATCH --exclusive
#SBATCH --time=1200
#SBATCH --mail-type=END
#SBATCH --mail-user=holger.schulz@durham.ac.uk
#SBATCH --array=0-12


source /etc/profile.d/modules.sh

module purge
module load aocc
#export FLAVOUR_NOCONFLICT=1
#module load gcc
module load likwid

#export OMP_NUM_THREADS=$SLURM_ARRAY_TASK_ID
#export OMP_PROC_BIND=close

unset OMP_NUM_THREADS

OUTDIR=$NOBACKUP/ccz4-regular-grid-AOCC

mkdir -p ${OUTDIR}


# NUMA stuff
declare -a arr=("M0" "M1" "M2" "M3" "M4" "M5" "M6" "M7" "M0@M1" "M0@M1@M2@M3" "M4@M5" "M4@M5@M6@M7" "M0@M1@M2@M3@M4@M5@M6@M7")


PINNING=${arr[$SLURM_ARRAY_TASK_ID]}
echo "PINNING: ${PINNING}"

for volume_size in 0.1 0.05 0.01 0.005 0.001
do
    for tt in fixed adaptive local
    do
        for threading_model in fuse-immediately-1 fuse-immediately-2 fuse-immediately-4 fuse-immediately-8 fuse-late-1 fuse-late-2 fuse-late-4 fuse-late-8
        do
            TAG=${volume_size}-${tt}-${threading_model}-${SLURM_ARRAY_TASK_ID}
            likwid-perfctr -f -o ${OUTDIR}/flops-${TAG}.csv -g FLOPS_DP -C ${PINNING} ./peano4-h-${volume_size}-${tt}-gpu --timeout 3600 --threading-model ${threading_model}  > ${OUTDIR}/output-${TAG}-threads.data
        done
    done
done    

#!/bin/bash
#SBATCH --job-name=regnofuA
#SBATCH -p multi
#SBATCH -N 1
#SBATCH --exclusive
#SBATCH --time=1200
#SBATCH --mail-type=END
#SBATCH --mail-user=holger.schulz@durham.ac.uk
#SBATCH --array=0-12


source /etc/profile.d/modules.sh

module purge
module load aocc
#export FLAVOUR_NOCONFLICT=1
#module load gcc
module load likwid

#export OMP_NUM_THREADS=$SLURM_ARRAY_TASK_ID
#export OMP_PROC_BIND=close

unset OMP_NUM_THREADS

OUTDIR=$NOBACKUP/ccz4-regular-grid-AOCC-nofusion

mkdir -p ${OUTDIR}


# NUMA stuff
declare -a arr=("M0" "M1" "M2" "M3" "M4" "M5" "M6" "M7" "M0@M1" "M0@M1@M2@M3" "M4@M5" "M4@M5@M6@M7" "M0@M1@M2@M3@M4@M5@M6@M7")


PINNING=${arr[$SLURM_ARRAY_TASK_ID]}
echo "PINNING: ${PINNING}"

for volume_size in 0.1 0.05 0.01 0.005 0.001
do
    for tt in fixed adaptive local
    do
        for threading_model in bsp native backfill
        do
            TAG=${volume_size}-${tt}-${threading_model}-${SLURM_ARRAY_TASK_ID}
            likwid-perfctr -f -o ${OUTDIR}/flops-${TAG}.csv -g FLOPS_DP -C ${PINNING} ./peano4-h-${volume_size}-${tt} --timeout 3600 --threading-model ${threading_model}  > ${OUTDIR}/output-${TAG}-threads.data
        done
    done
done

oneAPI

module load oneapi
export FLAVOUR_NOCONFLICT=1
module load gcc
module load likwid

./configure CC=icx CXX=icpx LDFLAGS="-fiopenmp -L/apps/developers/libraries/likwid/5.2.0/1/default -llikwid" CXXFLAGS="-g -Ofast -std=c++17 -fiopenmp -march=native -mtune=native -DLIKWID_PERFMON" --with-multithreading=omp --enable-exahype --enable-loadbalancing --enable-blockstructured

#!/bin/bash
#SBATCH --job-name=regfus
#SBATCH -p multi
#SBATCH -N 1
#SBATCH --exclusive
#SBATCH --time=1200
#SBATCH --mail-type=END
#SBATCH --mail-user=holger.schulz@durham.ac.uk
#SBATCH --array=0-12


source /etc/profile.d/modules.sh

module purge
module load oneapi
export FLAVOUR_NOCONFLICT=1
module load gcc
module load likwid

#export OMP_NUM_THREADS=$SLURM_ARRAY_TASK_ID
#export OMP_PROC_BIND=close

unset OMP_NUM_THREADS

OUTDIR=$NOBACKUP/ccz4-regular-grid-oneAPI

mkdir -p ${OUTDIR}


# NUMA stuff
declare -a arr=("M0" "M1" "M2" "M3" "M4" "M5" "M6" "M7" "M0@M1" "M0@M1@M2@M3" "M4@M5" "M4@M5@M6@M7" "M0@M1@M2@M3@M4@M5@M6@M7")


PINNING=${arr[$SLURM_ARRAY_TASK_ID]}
echo "PINNING: ${PINNING}"

for volume_size in 0.1 0.05 0.01 0.005 0.001
do
    for tt in fixed adaptive local
    do
        for threading_model in fuse-immediately-1 fuse-immediately-2 fuse-immediately-4 fuse-immediately-8 fuse-late-1 fuse-late-2 fuse-late-4 fuse-late-8
        do
            TAG=${volume_size}-${tt}-${threading_model}-${SLURM_ARRAY_TASK_ID}
            likwid-perfctr -f -o ${OUTDIR}/flops-${TAG}.csv -g FLOPS_DP -C ${PINNING} ./peano4-h-${volume_size}-${tt}-gpu --timeout 3600 --threading-model ${threading_model}  > ${OUTDIR}/output-${TAG}-threads.data
        done
    done
done    

#!/bin/bash
#SBATCH --job-name=regnofus
#SBATCH -p multi
#SBATCH -N 1
#SBATCH --exclusive
#SBATCH --time=1200
#SBATCH --mail-type=END
#SBATCH --mail-user=holger.schulz@durham.ac.uk
#SBATCH --array=0-12


source /etc/profile.d/modules.sh

module purge
module load oneapi
export FLAVOUR_NOCONFLICT=1
module load gcc
module load likwid

#export OMP_NUM_THREADS=$SLURM_ARRAY_TASK_ID
#export OMP_PROC_BIND=close

unset OMP_NUM_THREADS

OUTDIR=$NOBACKUP/ccz4-regular-grid-oneAPI-nofusion

mkdir -p ${OUTDIR}


# NUMA stuff
declare -a arr=("M0" "M1" "M2" "M3" "M4" "M5" "M6" "M7" "M0@M1" "M0@M1@M2@M3" "M4@M5" "M4@M5@M6@M7" "M0@M1@M2@M3@M4@M5@M6@M7")


PINNING=${arr[$SLURM_ARRAY_TASK_ID]}
echo "PINNING: ${PINNING}"

for volume_size in 0.1 0.05 0.01 0.005 0.001
do
    for tt in fixed adaptive local
    do
        for threading_model in bsp native backfill
        do
            TAG=${volume_size}-${tt}-${threading_model}-${SLURM_ARRAY_TASK_ID}
            likwid-perfctr -f -o ${OUTDIR}/flops-${TAG}.csv -g FLOPS_DP -C ${PINNING} ./peano4-h-${volume_size}-${tt} --timeout 3600 --threading-model ${threading_model}  > ${OUTDIR}/output-${TAG}-threads.data
        done
    done
done