/**
* \brief Initialize the DCMF ARMCI resources
*/
int ARMCIX_Init ()
{
DCMF_CriticalSection_enter(0);
DCMF_Messager_initialize ();
ARMCIX_DCMF_Connection_initialize ();
/* Determine request pool defaults */
int ARMCIX_DCMF_REQUESTPOOL_MAX = 1000;
ENV_Int (getenv ("ARMCIX_DCMF_REQUESTPOOL_MAX"), &ARMCIX_DCMF_REQUESTPOOL_MAX);
int ARMCIX_DCMF_REQUESTPOOL_INC = 0;
ENV_Int (getenv ("ARMCIX_DCMF_REQUESTPOOL_INC"), &ARMCIX_DCMF_REQUESTPOOL_INC);
ARMCIX_DCMF_request_initialize (ARMCIX_DCMF_REQUESTPOOL_MAX, ARMCIX_DCMF_REQUESTPOOL_INC);
ARMCIX_DCMF_Get_register ();
ARMCIX_DCMF_Put_register (__connection);
ARMCIX_DCMF_Acc_register (__connection);
ARMCIX_DCMF_Fence_register (__connection);
ARMCIX_DCMF_Rmw_register ();
/* Determine interrupt mode */
int interrupts = 1;
ENV_Bool (getenv ("DCMF_INTERRUPT"), &interrupts);
ENV_Bool (getenv ("DCMF_INTERRUPTS"), &interrupts);
DCMF_Configure_t config;
memset (&config, 0x00, sizeof(DCMF_Configure_t));
config.interrupts = (interrupts==0)?DCMF_INTERRUPTS_OFF:DCMF_INTERRUPTS_ON;
DCMF_Messager_configure (&config, &config);
DCMF_Messager_configure (NULL, &config);
//ARMCIX_DCMF_request_print ("after armcix_init");
DCMF_CriticalSection_exit(0);
return 0;
}
int A1D_Initialize()
{
int mpi_initialized, mpi_provided;
int mpi_status;
int i;
size_t bytes_in, bytes_out;
DCMF_Result dcmf_result;
DCMF_Configure_t dcmf_config;
DCMF_Memregion_t local_memregion;
/***************************************************
*
* configure MPI
*
***************************************************/
/* MPI has to be initialized for this implementation to work */
MPI_Initialized(&mpi_initialized);
assert(mpi_initialized==1);
/* MPI has to be thread-safe so that DCMF doesn't explode */
MPI_Query_thread(&mpi_provided);
assert(mpi_provided==MPI_THREAD_MULTIPLE);
/* have to use our own communicator for collectives to be proper */
mpi_status = MPI_Comm_dup(MPI_COMM_WORLD,&A1D_COMM_WORLD);
assert(mpi_status==0);
/* get my MPI rank */
mpi_status = MPI_Comm_rank(A1D_COMM_WORLD,&myrank);
assert(mpi_status==0);
/* get MPI world size */
mpi_status = MPI_Comm_size(A1D_COMM_WORLD,&mpi_size);
assert(mpi_status==0);
/* make sure MPI and DCMF agree */
assert(myrank==DCMF_Messager_rank());
assert(mpi_size==DCMF_Messager_size());
/* barrier before DCMF_Messager_configure to make sure MPI is ready everywhere */
mpi_status = MPI_Barrier(A1D_COMM_WORLD);
assert(mpi_status==0);
/***************************************************
*
* configure DCMF
*
***************************************************/
/* to be safe, but perhaps not necessary */
dcmf_config.thread_level = DCMF_THREAD_MULTIPLE;
#ifdef ACCUMULATE_IMPLEMENTED
/* interrupts required for accumulate only, Put/Get use DMA
* if accumulate not used, MPI will query environment for DCMF_INTERRUPTS */
dcmf_config.interrupts = DCMF_INTERRUPTS_ON;
#endif
/* reconfigure DCMF with interrupts on */
DCMF_CriticalSection_enter(0);
dcmf_result = DCMF_Messager_configure(&dcmf_config, &dcmf_config);
assert(dcmf_result==DCMF_SUCCESS);
DCMF_CriticalSection_exit(0);
/* barrier after DCMF_Messager_configure to make sure everyone has the new DCMF config */
mpi_status = MPI_Barrier(A1D_COMM_WORLD);
assert(mpi_status==0);
/***************************************************
*
* setup DCMF memregions
*
***************************************************/
/* allocate memregion list */
A1D_Memregion_list = malloc( mpi_size * sizeof(DCMF_Memregion_t) );
assert(A1D_Memregion_list != NULL);
/* allocate base pointer list */
A1D_Baseptr_list = malloc( mpi_size * sizeof(void*) );
assert(A1D_Memregion_list != NULL);
/* create memregions */
bytes_in = -1;
DCMF_CriticalSection_enter(0);
dcmf_result = DCMF_Memregion_create(&local_memregion,&bytes_out,bytes_in,NULL,0);
assert(dcmf_result==DCMF_SUCCESS);
DCMF_CriticalSection_exit(0);
/* exchange memregions because we don't use symmetry heap */
mpi_status = MPI_Allgather(&local_memregion,sizeof(DCMF_Memregion_t),MPI_BYTE,
A1D_Memregion_list,sizeof(DCMF_Memregion_t),MPI_BYTE,
A1D_COMM_WORLD);
assert(mpi_status==0);
/* destroy temporary local memregion */
DCMF_CriticalSection_enter(0);
dcmf_result = DCMF_Memregion_destroy(&local_memregion);
assert(dcmf_result==DCMF_SUCCESS);
DCMF_CriticalSection_exit(0);
/* check for valid memregions */
DCMF_CriticalSection_enter(0);
for (i = 0; i < mpi_size; i++)
{
dcmf_result = DCMF_Memregion_query(&A1D_Memregion_list[i],
&bytes_out,
&A1D_Baseptr_list[i]);
assert(dcmf_result==DCMF_SUCCESS);
}
DCMF_CriticalSection_exit(0);
#ifdef FLUSH_IMPLEMENTED
/***************************************************
*
* setup flush list(s)
*
***************************************************/
/* allocate Put list */
A1D_Put_flush_list = malloc( mpi_size * sizeof(int) );
assert(A1D_Put_flush_list != NULL);
#ifdef ACCUMULATE_IMPLEMENTED
/* allocate Acc list */
A1D_Send_flush_list = malloc( mpi_size * sizeof(int) );
assert(A1D_Send_flush_list != NULL);
#endif
#endif
/***************************************************
*
* define null callback
*
***************************************************/
A1D_Nocallback.function = NULL;
A1D_Nocallback.clientdata = NULL;
return(0);
}
/**
* \brief Initialize the DCMF ARMCI resources
*/
int ARMCIX_Init ()
{
DCMF_CriticalSection_enter(0);
DCMF_Messager_initialize ();
ARMCIX_DCMF_Connection_initialize ();
/* Determine request pool defaults */
int ARMCIX_DCMF_REQUESTPOOL_MAX = 1000;
ENV_Int (getenv ("ARMCIX_DCMF_REQUESTPOOL_MAX"), &ARMCIX_DCMF_REQUESTPOOL_MAX);
int ARMCIX_DCMF_REQUESTPOOL_INC = 0;
ENV_Int (getenv ("ARMCIX_DCMF_REQUESTPOOL_INC"), &ARMCIX_DCMF_REQUESTPOOL_INC);
ARMCIX_DCMF_request_initialize (ARMCIX_DCMF_REQUESTPOOL_MAX, ARMCIX_DCMF_REQUESTPOOL_INC);
ARMCIX_DCMF_Get_register ();
ARMCIX_DCMF_Put_register (__connection);
ARMCIX_DCMF_Acc_register (__connection);
ARMCIX_DCMF_Fence_register (__connection);
ARMCIX_DCMF_Rmw_register ();
/* Initializer helper thread or configure interrupt mode */
int interrupts = 0;
ENV_Bool (getenv ("DCMF_INTERRUPT"), &interrupts);
ENV_Bool (getenv ("DCMF_INTERRUPTS"), &interrupts);
//fprintf(stdout,"interrupts = %d\n",interrupts);
/*if (interrupts==1){
if( 0==DCMF_Messager_rank() ) fprintf(stdout,"DCMF interrupts ON\n");
} else {
if( 0==DCMF_Messager_rank() ) fprintf(stdout,"DCMF interrupts OFF\n");
}*/
if (interrupts==0) {
int ret = pthread_create(&armcix_advance_thread, NULL, armcix_advance, NULL);
if ( ret != 0 ) {
if( 0==DCMF_Messager_rank() ) fprintf(stdout,"pthread_create failed\n");
armcix_advance_active = 0;
} else {
if( 0==DCMF_Messager_rank() ) fprintf(stdout,"pthread_create succeeded\n");
armcix_advance_active = 1;
}
}
DCMF_Configure_t config;
memset (&config, 0x00, sizeof(DCMF_Configure_t));
config.interrupts = (interrupts==0)?DCMF_INTERRUPTS_OFF:DCMF_INTERRUPTS_ON;
DCMF_Messager_configure (&config, &config);
DCMF_Messager_configure (NULL, &config);
//ARMCIX_DCMF_request_print ("after armcix_init");
DCMF_CriticalSection_exit(0);
return 0;
}
int main(int argc, void* argv[])
{
DCMF_Configure_t config;
config.thread_level = DCMF_THREAD_MULTIPLE;
DCMF_Messager_initialize();
DCMF_Messager_configure(&config, &config);
init();
if (nranks != (THREAD_NUM + 1))
{
printf("This test requires only %d processes \n", (THREAD_NUM + 1));
fflush(stdout);
return -1;
}
barrier_init(DCMF_DEFAULT_GLOBALBARRIER_PROTOCOL);
control_init(DCMF_DEFAULT_CONTROL_PROTOCOL, DCMF_DEFAULT_NETWORK);
memregion_init(LOCAL_MAX_BUF_SIZE * THREAD_NUM);
get_init(DCMF_DEFAULT_PUT_PROTOCOL, DCMF_TORUS_NETWORK);
source = (char *) malloc(LOCAL_MAX_BUF_SIZE * THREAD_NUM);
target = (char *) malloc(LOCAL_MAX_BUF_SIZE * THREAD_NUM);
send_init(DCMF_DEFAULT_SEND_PROTOCOL, DCMF_TORUS_NETWORK);
int status;
long i;
if (myrank == 0)
{
pthread_t threads[THREAD_NUM];
pthread_barrier_init(&ptbarrier, NULL, THREAD_NUM);
pthread_barrier_init(&ptbarrier1, NULL, THREAD_NUM);
for (i = 0; i < THREAD_NUM; i++)
{
pthread_create(&threads[i], NULL, mrate_test, (void *) i);
}
for (i = 0; i < THREAD_NUM; i++)
{
pthread_join(threads[i], (void *) &status);
}
}
else
{
snd_rcv_active += LOCAL_ITERATIONS;
while (snd_rcv_active > 0)
DCMF_Messager_advance();
}
barrier();
DCMF_Messager_finalize();
if (myrank == 0)
{
printf("Benchmark Complete \n");
fflush(stdout);
}
return (0);
}
