SCOREC RHEL9 Remus - SCOREC/pumi-pic GitHub Wiki

Environment File

export root=/space/castia5/remus-pumi-pic
module use /opt/scorec/spack/rhel9/v0201_4/lmod/linux-rhel9-x86_64/Core/
module load gcc/12.3.0-iil3lno
module load mpich/4.1.1-xpoyz4t
module load cuda/12.1.1-zxa4msk
module load netcdf-c/4.9.2-2ilqxr3
module load cmake/3.26.3-2duxfcd
module load hdf5
module load netcdf-fortran

export MPICH_CXX=$root/kokkos/bin/nvcc_wrapper
export gpu_option=ON
export kokkos_cpu=-DKokkos_ENABLE_OPENMP=OFF
export kokkos_gpu=-DKokkos_ARCH_PASCAL61=ON
export omegah_gpu=-DOmega_h_CUDA_ARCH=61
export ftn_compiler=gfortran
export cxx_compiler=mpicxx
export c_compiler=mpicc

Install File

#!/bin/bash -e

# mkdir $root
cd $root

export CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:\
$root/build-kokkos/install:\
$root/build-omegah/install:\
$root/build-cabana/install:\
$root/build-engpar/install:\
$root/build-pumipic/install

#kokkos
rm kokkos -rf
git clone -b 4.5.00 https://github.com/kokkos/kokkos.git
rm build-kokkos -rf
cmake -S kokkos -B build-kokkos \
  -DCMAKE_INSTALL_PREFIX=build-kokkos/install \
  -DCMAKE_CXX_COMPILER=$MPICH_CXX \
  $kokkos_gpu \
  -DKokkos_ENABLE_SERIAL=ON \
  $kokkos_cpu \
  -DKokkos_ENABLE_CUDA=$gpu_option \
  -DKokkos_ENABLE_CUDA_LAMBDA=$gpu_option \
  -DKokkos_ENABLE_DEBUG=off
cmake --build build-kokkos -j 24 --target install

#omegah
rm omega_h -rf
git clone -b scorec-v10.8.4 https://github.com/SCOREC/omega_h.git
rm build-omegah -rf
cmake -S omega_h -B build-omegah \
  -DCMAKE_INSTALL_PREFIX=build-omegah/install \
  -DCMAKE_BUILD_TYPE=Release \
  -DBUILD_SHARED_LIBS=OFF \
  -DOmega_h_USE_Kokkos=ON \
  -DOmega_h_USE_CUDA=$gpu_option \
  $omegah_gpu \
  -DOmega_h_USE_MPI=on  \
  -DBUILD_TESTING=off  \
  -DCMAKE_C_COMPILER=$c_compiler \
  -DCMAKE_CXX_COMPILER=$cxx_compiler
cmake --build build-omegah -j 24 --target install

#engpar
rm EnGPar -rf
git clone https://github.com/SCOREC/EnGPar.git
unset MPICH_CXX #don't want nvcc_wrapper for engpar
rm build-engpar -rf
cmake -S EnGPar -B build-engpar \
  -DCMAKE_INSTALL_PREFIX=build-engpar/install \
  -DCMAKE_BUILD_TYPE=Release \
  -DCMAKE_C_COMPILER=$c_compiler \
  -DCMAKE_CXX_COMPILER=$cxx_compiler \
  -DCMAKE_CXX_FLAGS="-std=c++11" \
  -DENABLE_PARMETIS=OFF \
  -DENABLE_PUMI=OFF \
  -DIS_TESTING=OFF
cmake --build build-engpar -j 24 --target install
export MPICH_CXX=$root/kokkos/bin/nvcc_wrapper #restore use of nvcc_wrapper

#cabana
rm cabana -rf
git clone -b 0.6.1 https://github.com/ECP-copa/Cabana.git cabana
rm build-cabana -rf
cmake -S cabana -B build-cabana \
  -DCMAKE_INSTALL_PREFIX=build-cabana/install \
  -DCMAKE_BUILD_TYPE=Release \
  -DCMAKE_CXX_COMPILER=$MPICH_CXX \
  -DCabana_REQUIRE_HDF5=OFF \
  -DCabana_ENABLE_TESTING=OFF \
  -DCabana_ENABLE_EXAMPLES=OFF
cmake --build build-cabana -j 24 --target install

# pumipic
rm pumi-pic -rf
git clone --recursive https://github.com/SCOREC/pumi-pic
cd pumi-pic && git checkout 13fca71cb02dc2503697ee3d00683f6e1a7fac9e && cd -
rm build-pumipic -rf
cmake -S pumi-pic -B build-pumipic \
  -DCMAKE_INSTALL_PREFIX=build-pumipic/install \
  -DCMAKE_BUILD_TYPE=Release \
  -DCMAKE_CXX_COMPILER=$cxx_compiler \
  -DENABLE_CABANA=ON \
  -DTEST_DATA_DIR=$root/pumi-pic/pumipic-data \
  -DIS_TESTING=on \
  -DPS_IS_TESTING=on
cmake --build build-pumipic -j 24 --target install