Mira and Cetus - PHASTA/phasta GitHub Wiki

Tau

Documentation

2015 Mira Performance Bootcamp presentation

Mira/Cetus/Vesta Tau User Guide

Setup

Run the following commands in each terminal/session.

soft add +tau-latest
export TAU_MAKEFILE=$TAU/Makefile.tau-bgqtimers-papi-mpi-pdt

Building

Create the toolchain file bgq-xl-tau.cmake with the following contents

set(CMAKE_SYSTEM_NAME BlueGeneQ-static)

set(COMPILER_SEARCH_PATHS 
/soft/perftools/tau/tau_latest/ppc64/bin/
/bgsys/drivers/ppcfloor/comm/xl/bin 
/soft/compilers/ibmcmp-aug2015/vac/bg/12.1/bin/
/soft/compilers/ibmcmp-aug2015/xlf/bg/14.1/bin/
/soft/compilers/ibmcmp-may2015/vacpp/bg/12.1/bin/)

find_program(CMAKE_C_COMPILER       tau_cc.sh    ${COMPILER_SEARCH_PATHS} )
find_program(CMAKE_CXX_COMPILER     tau_cxx.sh   ${COMPILER_SEARCH_PATHS} )
find_program(CMAKE_Fortran_COMPILER tau_f90.sh  ${COMPILER_SEARCH_PATHS} )

SET_PROPERTY(GLOBAL PROPERTY TARGET_SUPPORTS_SHARED_LIBS FALSE)
SET_PROPERTY(GLOBAL PROPERTY TARGET_ARCHIVES_MAY_BE_SHARED_LIBS FALSE)

set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)

#The combination of tau's use of the mpi compiler wrappers and our
# use of MPI_*_libraries from find_package(mpi) results in
# several 'multiple symbol definition' errors when linking phastaIC.exe.
#Override the variables set by find_package(mpi) to avoid find_package(mpi) 
# autodetection.  Setting the libraries to 'm' is a hack - FindMPI.cmake 
# does not want those vars to be empty strings.
set(MPI_Fortran_COMPILER mpixlf90)
set(MPI_Fortran_INCLUDE_PATH "/bgsys/drivers/V1R2M2/ppc64")
set(MPI_Fortran_LIBRARIES "c")
set(MPI_C_COMPILER mpixlc)
set(MPI_C_INCLUDE_PATH "/bgsys/drivers/V1R2M2/ppc64/comm/include")
set(MPI_C_LIBRARIES "c")
set(MPI_CXX_COMPILER mpixlcxx)
set(MPI_CXX_INCLUDE_PATH "/bgsys/drivers/V1R2M2/ppc64/comm/lib/xl")
set(MPI_CXX_LIBRARIES "c")

Create a cmake script config-xl-tau.sh with the following contents:

#!/bin/bash -ex
opt=" -O2 -optKeepFiles -tau_options='-optTauSelectFile=/path/to/selectfile' "
les=/path/to/libles.a
toolchain=/path/to/bgq-xl-tau.cmake

cmake .. \
--debug-output \
-DCMAKE_TOOLCHAIN_FILE=$toolchain \
-DCMAKE_BUILD_TYPE="Release" \
-DCMAKE_C_FLAGS_RELEASE="$opt" \
-DCMAKE_CXX_FLAGS_RELEASE="$opt" \
-DCMAKE_Fortran_FLAGS_RELEASE="$opt" \
\
-DPHASTA_INCOMPRESSIBLE=ON \
-DPHASTA_COMPRESSIBLE=ON \
-DLESLIB=$les

Configure and make:

   # this will fail if TAU_MAKEFILE is not set - see setup above
   chmod +x config-xl-tau.sh
   mkdir build-xl-tau
   cd build-xl-tau
   ./config-xl-tau.sh
   make #and wait

Running

Submit the job and run as you would normally. Optionally, Tau variables can be passed to qsub (can't get it working with runjob....) to control its behavior. For example, to merge profile data into a single file (strongly recommended) pass the following environment setting to qsub:

--env TAU_PROFILE_FORMAT="merged"

A sample execution script is below:

#!/bin/bash -ex
project=SC_Architectures
email=you@domain
processes=$((2**16))
ppn=64
nodes=$((processes/ppn))
startingtimestep=76800
#procscasedir=65536-procs_case-SyncIO-64-NoParma
procscasedir=65536-procs_case-SyncIO-64-Parma
exe=/home/cwsmith/projects/mrPhasta/build-tau/bin/phastaIC.exe
BG_OPTS="BG_SHAREDMEMSIZE=64MB:PAMID_VERBOSE=1"
TAU_OPTS="TAU_PROFILE_FORMAT=merged:TAU_METRICS=BGQ_TIMERS:PAPI_L1_DCM:PAPI_NATIVE_PEVT_L2_MISSES_SLICE"
PHASTA_OPTS="PHASTA_CONFIG=/gpfs/mira-fs0/projects/UnsAdaptCFD_tesp/cwsmith/phastaParmaDec2015"
ln -snf $procscasedir $processes-procs_case
echo $startingtimestep > $processes-procs_case/numstart.dat
echo "processes=$processes ppn=$ppn nodes=$nodes startingtimestep=$startingtimestep "
qsub -t 30 -n $nodes --mode c${ppn} -A $project -M $email --env $TAU_OPTS --env $PHASTA_OPTS --env $BG_OPTS $exe solver.mr.inp

Analysis

The tau profile data will be written to the N-procs_case directory as tauprofile.xml .

Run the following steps on the cooley front end node.

Setup the environment

For some reason there is no tau soft package on cooley, so do the following before running paraprof:

export PATH=${PATH}:\
/soft/perftools/tau/pdt_latest/x86_64/bin:\
/soft/perftools/tau/tau_latest/x86_64/bin
export TAU=/soft/perftools/tau/tau_latest/x86_64/lib

Compress the profile data

Write the compressed profile data to /path/to/tauprofile.ppk

paraprof --pack /path/to/tauprofile.ppk /path/to/N-procs_case/tauprofile.xml

The compressed file will load faster in paraprof and consume less disk space.

Run paraprof

paraprof /path/to/tauprofile.ppk