179 lines
5.3 KiB
C++
179 lines
5.3 KiB
C++
/*
|
|
//@HEADER
|
|
// ************************************************************************
|
|
//
|
|
// Kokkos v. 2.0
|
|
// Copyright (2014) Sandia Corporation
|
|
//
|
|
// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
|
|
// the U.S. Government retains certain rights in this software.
|
|
//
|
|
// Redistribution and use in source and binary forms, with or without
|
|
// modification, are permitted provided that the following conditions are
|
|
// met:
|
|
//
|
|
// 1. Redistributions of source code must retain the above copyright
|
|
// notice, this list of conditions and the following disclaimer.
|
|
//
|
|
// 2. Redistributions in binary form must reproduce the above copyright
|
|
// notice, this list of conditions and the following disclaimer in the
|
|
// documentation and/or other materials provided with the distribution.
|
|
//
|
|
// 3. Neither the name of the Corporation nor the names of the
|
|
// contributors may be used to endorse or promote products derived from
|
|
// this software without specific prior written permission.
|
|
//
|
|
// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
|
|
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
|
|
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
|
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
|
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
|
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
|
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
|
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
|
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
//
|
|
// Questions? Contact Christian R. Trott (crtrott@sandia.gov)
|
|
//
|
|
// ************************************************************************
|
|
//@HEADER
|
|
*/
|
|
|
|
#if 0
|
|
|
|
#include <cstdlib>
|
|
#include <cstring>
|
|
|
|
#include <ParallelMachine.hpp>
|
|
|
|
#include <Kokkos_Core.hpp>
|
|
|
|
#if ! defined( KOKKOS_ENABLE_MPI )
|
|
#define MPI_COMM_NULL 0
|
|
#endif
|
|
|
|
//------------------------------------------------------------------------
|
|
|
|
namespace Parallel {
|
|
|
|
Machine::Machine( int * argc , char *** argv )
|
|
: m_mpi_comm( MPI_COMM_NULL )
|
|
, m_mpi_size(0)
|
|
, m_mpi_rank(0)
|
|
, m_mpi_gpu(0)
|
|
{
|
|
|
|
#if defined( KOKKOS_ENABLE_CUDA )
|
|
//------------------------------------
|
|
// Might be using a Cuda aware version of MPI.
|
|
// Must select Cuda device before initializing MPI.
|
|
{
|
|
int i = 1 ;
|
|
for ( ; i < *argc && strcmp((*argv)[i],"mpi_cuda") ; ++i );
|
|
|
|
if ( i < *argc ) {
|
|
// Determine, if possible, what will be the node-local
|
|
// rank of the MPI process once MPI has been initialized.
|
|
// This rank is needed to set the Cuda device before 'mvapich'
|
|
// is initialized.
|
|
|
|
const char * const mvapich_local_rank = getenv("MV2_COMM_WORLD_LOCAL_RANK");
|
|
const char * const slurm_local_rank = getenv("SLURM_LOCALID");
|
|
|
|
const int pre_mpi_local_rank =
|
|
0 != mvapich_local_rank ? atoi( mvapich_local_rank ) : (
|
|
0 != slurm_local_rank ? atoi( slurm_local_rank ) : (
|
|
-1 ) );
|
|
|
|
if ( 0 <= pre_mpi_local_rank ) {
|
|
|
|
const int ngpu = Kokkos::Cuda::detect_device_count();
|
|
|
|
const int cuda_device_rank = pre_mpi_local_rank % ngpu ;
|
|
|
|
Kokkos::Cuda::initialize( Kokkos::Cuda::SelectDevice( cuda_device_rank ) );
|
|
|
|
m_mpi_gpu = 1 ;
|
|
}
|
|
}
|
|
}
|
|
#endif
|
|
|
|
//------------------------------------
|
|
|
|
#if defined( KOKKOS_ENABLE_MPI )
|
|
MPI_Init( argc , argv );
|
|
m_mpi_comm = MPI_COMM_WORLD ;
|
|
MPI_Comm_size( m_mpi_comm , & m_mpi_size );
|
|
MPI_Comm_rank( m_mpi_comm , & m_mpi_rank );
|
|
#endif
|
|
|
|
// Query hwloc after MPI initialization to allow MPI binding:
|
|
//------------------------------------
|
|
// Request to use host device:
|
|
{
|
|
int i = 1 ;
|
|
for ( ; i < *argc && strcmp((*argv)[i],"host") ; ++i );
|
|
|
|
if ( i < *argc ) {
|
|
|
|
unsigned team_count = Kokkos::hwloc::get_available_numa_count();
|
|
unsigned threads_per_team = Kokkos::hwloc::get_available_cores_per_numa() *
|
|
Kokkos::hwloc::get_available_threads_per_core();
|
|
|
|
if ( i + 2 < *argc ) {
|
|
team_count = atoi( (*argv)[i+1] );
|
|
threads_per_team = atoi( (*argv)[i+2] );
|
|
}
|
|
|
|
Kokkos::Threads::initialize( team_count * threads_per_team );
|
|
}
|
|
}
|
|
|
|
#if defined( KOKKOS_ENABLE_CUDA )
|
|
//------------------------------------
|
|
// Request to use Cuda device and not already initialized.
|
|
if ( ! m_mpi_gpu ) {
|
|
int i = 1 ;
|
|
for ( ; i < *argc && strcmp((*argv)[i],"mpi_cuda") && strcmp((*argv)[i],"cuda") ; ++i );
|
|
|
|
if ( i < *argc ) {
|
|
|
|
const int ngpu = Kokkos::Cuda::detect_device_count();
|
|
|
|
const int cuda_device_rank = m_mpi_rank % ngpu ;
|
|
|
|
Kokkos::Cuda::initialize( Kokkos::Cuda::SelectDevice( cuda_device_rank ) );
|
|
}
|
|
}
|
|
#endif
|
|
|
|
}
|
|
|
|
Machine::~Machine()
|
|
{
|
|
Kokkos::Threads::finalize();
|
|
#if defined( KOKKOS_ENABLE_CUDA )
|
|
Kokkos::Cuda::finalize();
|
|
#endif
|
|
#if defined( KOKKOS_ENABLE_MPI )
|
|
MPI_Finalize();
|
|
#endif
|
|
}
|
|
|
|
void Machine::print_configuration( std::ostream & msg ) const
|
|
{
|
|
msg << "MPI [ " << m_mpi_rank << " / " << m_mpi_size << " ]" << std::endl ;
|
|
Kokkos::Threads::print_configuration( msg );
|
|
#if defined( KOKKOS_ENABLE_CUDA )
|
|
Kokkos::Cuda::print_configuration( msg );
|
|
#endif
|
|
}
|
|
|
|
}
|
|
|
|
#endif /* #if 0 */
|
|
|