Mira and Cetus - PHASTA/phasta GitHub Wiki
Tau
Documentation
2015 Mira Performance Bootcamp presentation
Mira/Cetus/Vesta Tau User Guide
Setup
Run the following commands in each terminal/session.
soft add +tau-latest
export TAU_MAKEFILE=$TAU/Makefile.tau-bgqtimers-papi-mpi-pdt
Building
Create the toolchain file bgq-xl-tau.cmake with the following contents
set(CMAKE_SYSTEM_NAME BlueGeneQ-static)
set(COMPILER_SEARCH_PATHS
/soft/perftools/tau/tau_latest/ppc64/bin/
/bgsys/drivers/ppcfloor/comm/xl/bin
/soft/compilers/ibmcmp-aug2015/vac/bg/12.1/bin/
/soft/compilers/ibmcmp-aug2015/xlf/bg/14.1/bin/
/soft/compilers/ibmcmp-may2015/vacpp/bg/12.1/bin/)
find_program(CMAKE_C_COMPILER tau_cc.sh ${COMPILER_SEARCH_PATHS} )
find_program(CMAKE_CXX_COMPILER tau_cxx.sh ${COMPILER_SEARCH_PATHS} )
find_program(CMAKE_Fortran_COMPILER tau_f90.sh ${COMPILER_SEARCH_PATHS} )
SET_PROPERTY(GLOBAL PROPERTY TARGET_SUPPORTS_SHARED_LIBS FALSE)
SET_PROPERTY(GLOBAL PROPERTY TARGET_ARCHIVES_MAY_BE_SHARED_LIBS FALSE)
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
#The combination of tau's use of the mpi compiler wrappers and our
# use of MPI_*_libraries from find_package(mpi) results in
# several 'multiple symbol definition' errors when linking phastaIC.exe.
#Override the variables set by find_package(mpi) to avoid find_package(mpi)
# autodetection. Setting the libraries to 'm' is a hack - FindMPI.cmake
# does not want those vars to be empty strings.
set(MPI_Fortran_COMPILER mpixlf90)
set(MPI_Fortran_INCLUDE_PATH "/bgsys/drivers/V1R2M2/ppc64")
set(MPI_Fortran_LIBRARIES "c")
set(MPI_C_COMPILER mpixlc)
set(MPI_C_INCLUDE_PATH "/bgsys/drivers/V1R2M2/ppc64/comm/include")
set(MPI_C_LIBRARIES "c")
set(MPI_CXX_COMPILER mpixlcxx)
set(MPI_CXX_INCLUDE_PATH "/bgsys/drivers/V1R2M2/ppc64/comm/lib/xl")
set(MPI_CXX_LIBRARIES "c")
Create a cmake script config-xl-tau.sh with the following contents:
#!/bin/bash -ex
opt=" -O2 -optKeepFiles -tau_options='-optTauSelectFile=/path/to/selectfile' "
les=/path/to/libles.a
toolchain=/path/to/bgq-xl-tau.cmake
cmake .. \
--debug-output \
-DCMAKE_TOOLCHAIN_FILE=$toolchain \
-DCMAKE_BUILD_TYPE="Release" \
-DCMAKE_C_FLAGS_RELEASE="$opt" \
-DCMAKE_CXX_FLAGS_RELEASE="$opt" \
-DCMAKE_Fortran_FLAGS_RELEASE="$opt" \
\
-DPHASTA_INCOMPRESSIBLE=ON \
-DPHASTA_COMPRESSIBLE=ON \
-DLESLIB=$les
Configure and make:
# this will fail if TAU_MAKEFILE is not set - see setup above
chmod +x config-xl-tau.sh
mkdir build-xl-tau
cd build-xl-tau
./config-xl-tau.sh
make #and wait
Running
Submit the job and run as you would normally. Optionally, Tau variables can be passed to qsub (can't get it working with runjob....) to control its behavior. For example, to merge profile data into a single file (strongly recommended) pass the following environment setting to qsub:
--env TAU_PROFILE_FORMAT="merged"
A sample execution script is below:
#!/bin/bash -ex
project=SC_Architectures
email=you@domain
processes=$((2**16))
ppn=64
nodes=$((processes/ppn))
startingtimestep=76800
#procscasedir=65536-procs_case-SyncIO-64-NoParma
procscasedir=65536-procs_case-SyncIO-64-Parma
exe=/home/cwsmith/projects/mrPhasta/build-tau/bin/phastaIC.exe
BG_OPTS="BG_SHAREDMEMSIZE=64MB:PAMID_VERBOSE=1"
TAU_OPTS="TAU_PROFILE_FORMAT=merged:TAU_METRICS=BGQ_TIMERS:PAPI_L1_DCM:PAPI_NATIVE_PEVT_L2_MISSES_SLICE"
PHASTA_OPTS="PHASTA_CONFIG=/gpfs/mira-fs0/projects/UnsAdaptCFD_tesp/cwsmith/phastaParmaDec2015"
ln -snf $procscasedir $processes-procs_case
echo $startingtimestep > $processes-procs_case/numstart.dat
echo "processes=$processes ppn=$ppn nodes=$nodes startingtimestep=$startingtimestep "
qsub -t 30 -n $nodes --mode c${ppn} -A $project -M $email --env $TAU_OPTS --env $PHASTA_OPTS --env $BG_OPTS $exe solver.mr.inp
Analysis
The tau profile data will be written to the N-procs_case directory as tauprofile.xml .
Run the following steps on the cooley front end node.
Setup the environment
For some reason there is no tau soft package on cooley, so do the following before running paraprof:
export PATH=${PATH}:\
/soft/perftools/tau/pdt_latest/x86_64/bin:\
/soft/perftools/tau/tau_latest/x86_64/bin
export TAU=/soft/perftools/tau/tau_latest/x86_64/lib
Compress the profile data
Write the compressed profile data to /path/to/tauprofile.ppk
paraprof --pack /path/to/tauprofile.ppk /path/to/N-procs_case/tauprofile.xml
The compressed file will load faster in paraprof and consume less disk space.
Run paraprof
paraprof /path/to/tauprofile.ppk