void Simulation::communicate(std::vector<Messenger> messengers)
{
for (auto messenger : messengers)
bsp_send(messenger.d_p, messenger.d_tag, &messenger.d_src, sizeof(double));
bsp_sync();
MCBSP_NUMMSG_TYPE nmessages = 0;
MCBSP_BYTESIZE_TYPE nbytes = 0;
bsp_qsize(&nmessages, &nbytes);
for (MCBSP_NUMMSG_TYPE n = 0; n < nmessages; ++n)
{
size_t i[2];
MCBSP_BYTESIZE_TYPE status;
bsp_get_tag(&status,&i); // i[0] = idx, i[1] = dir
if (status > 0)
{
double distribution = 0;
bsp_move(&distribution, sizeof(double));
d_domain->nodes[i[0]].distributions[i[1]].nextValue = distribution;
}
}
// bsp_sync();
}
void spmd( void ) {
//parallel over three processes
bsp_begin( 3 );
//test bsp_push_reg (results in next superstep)
size_t localInt;
bsp_push_reg( &localInt, sizeof( size_t ) );
checkLocalIntAddress[ bsp_pid() ] = &localInt;
//check pid/nprocs, both using primitives as well as manually
checkPcount[ bsp_pid() ] = (size_t)(bsp_nprocs());
pthread_mutex_lock( &test_mutex );
check++;
checkP[ bsp_pid() ] = true;
pthread_mutex_unlock( &test_mutex );
//nobody should be at superstep 0
if( superstep == 1 )
superstepOK = false;
//test barrier synchronisation
bsp_sync();
//note someone is at superstep 1
superstep = 1;
//check bsp_time
if( bsp_time() <= 0 )
bsp_abort( "FAILURE \t bsp_time returned 0 or less!\n" );
//set up a pop_reg, but should only take effect after the next sync
//(testing the push_reg after this statement thus provides a free test)
bsp_pop_reg( &localInt );
struct mcbsp_thread_data * const data = pthread_getspecific( mcbsp_internal_thread_data );
if( data->localsToRemove.top != 1 || data->localsToRemove.cap != 16 ||
*((void**)(data->localsToRemove.array)) != (void*)&localInt ) {
fprintf( stderr, "FAILURE \t bsp_pop_reg did not push entry on the to-remove stack (%p != %p)!\n",
*((void**)(data->localsToRemove.array)), (void*)&localInt );
mcbsp_util_fatal();
}
//check push_reg
for( unsigned char i=0; i<3; ++i ) {
if( checkLocalIntAddress[ i ] != mcbsp_util_address_table_get( &(data->init->global2local), 0, i )->address ) {
fprintf( stderr, "FAILURE \t bsp_push_reg did not register correct address!\n" );
mcbsp_util_fatal();
}
}
bsp_sync();
//check pop_reg
for( unsigned char i=0; i<3; ++i ) {
if( mcbsp_util_address_table_get( &(data->init->global2local), 0, i ) != NULL ||
data->localC != 0 ) {
fprintf( stderr, "FAILURE \t bsp_pop_reg did not de-register correctly (entry=%p)!\n",
mcbsp_util_address_table_get( &(data->init->global2local), 0, i )->address );
mcbsp_util_fatal();
}
//localInt = *(size_t*)mcbsp_util_stack_pop( &(data->removedGlobals) );
}
bsp_sync();
//going to test communication primitives on the following area
size_t commTest[ 3 ];
commTest[ 0 ] = commTest[ 1 ] = ((size_t)bsp_pid());
commTest[ 2 ] = (size_t)(bsp_nprocs());
bsp_push_reg( &commTest, 3 * sizeof( size_t ) );
//make push valid
bsp_sync();
//after this put, commTest[ 0 ] should equal bsp_pid, commTest[ 1, 2 ] should equal bsp_pid-1 mod bsp_nprocs
bsp_put( (bsp_pid() + 1) % bsp_nprocs(), &commTest, &commTest, sizeof( size_t ), 2*sizeof( size_t) );
commTest[ 2 ] = ULONG_MAX; //this should not influence the result after sync.
//test behind-the-scenes
const struct mcbsp_util_stack queue = data->queues[ (bsp_pid() + 1) % bsp_nprocs() ];
size_t predicted_cap = predictCap( sizeof( struct mcbsp_message ) + 2 * sizeof( size_t) );
if( queue.cap != predicted_cap || queue.top != sizeof( struct mcbsp_message ) + 2 * sizeof( size_t) || queue.size != sizeof( struct mcbsp_message ) ) {
fprintf( stderr, "FAILURE \t bsp_put did not adapt the communication queue as expected!\n(cap = %ld, top = %ld, size = %ld)\n",
(size_t)queue.cap, (size_t)queue.top, (size_t)queue.size );
mcbsp_util_fatal();
}
const struct mcbsp_message request = *((struct mcbsp_message*) ((char*)queue.array + queue.top - sizeof( struct mcbsp_message )) );
if( request.length != 2 * sizeof( size_t) ) {
fprintf( stderr, "FAILURE \t bsp_put did not push a request of the expected length!\n(length = %ld)\n", (size_t)request.length );
mcbsp_util_fatal();
}
const size_t * const chk_array = (size_t*) ((char*)queue.array + queue.top - sizeof( struct mcbsp_message ) - 2 * sizeof( size_t ));
if( chk_array[ 0 ] != ((size_t)bsp_pid()) || chk_array[ 1 ] != ((size_t)bsp_pid()) ) {
fprintf( stderr, "FAILURE \t bsp_put did not push an expected communication request!\n" );
mcbsp_util_fatal();
}
//note there is no easy way to check request.destination; the top-level BSP test will handle that one
bsp_sync();
//test for the above expectation after bsp_put, namely
//commTest[ 0 ] should equal bsp_pid, commTest[ 1, 2 ] should equal bsp_pid-1 mod bsp_nprocs
if( commTest[ 0 ] != ((size_t)bsp_pid()) ||
commTest[ 1 ] != (size_t)((bsp_pid()+bsp_nprocs()-1)%bsp_nprocs()) ||
commTest[ 2 ] != (size_t)((bsp_pid()+bsp_nprocs()-1)%bsp_nprocs())
) {
fprintf( stderr, "FAILURE \t array after bsp_put is not as expected! (%d: %ld %ld %ld))\n", bsp_pid(), commTest[ 0 ], commTest[ 1 ], commTest[ 2 ] );
mcbsp_util_fatal();
}
//do a get on the next processor on the last element of commTest
bsp_get( (bsp_pid() + 1) % bsp_nprocs(), &commTest, 2 * sizeof( size_t ), &(commTest[ 2 ]), sizeof( size_t ) );
//fill the expected value after the get to test non-buffering
commTest[ 2 ] = ((size_t)bsp_pid());
//communicate
bsp_sync();
//commTest[ 0 ] should equal bsp_pid, commTest[ 1 ] should equal bsp_pid-1, commTest[ 2 ] should be bsp_pid+1
if( commTest[ 0 ] != ((size_t)bsp_pid()) ||
commTest[ 1 ] != (size_t)((bsp_pid()+bsp_nprocs() - 1)%bsp_nprocs())
) {
fprintf( stderr, "FAILURE \t start of array after bsp_get changed! (%d: %ld %ld %ld))\n", bsp_pid(), commTest[ 0 ], commTest[ 1 ], commTest[ 2 ] );
mcbsp_util_fatal();
}
if( commTest[ 2 ] != (size_t)((bsp_pid()+bsp_nprocs() + 1)%bsp_nprocs()) ) {
fprintf( stderr, "FAILURE \t last element of array after bsp_get erroneous! (%d: %ld %ld %ld))\n", bsp_pid(), commTest[ 0 ], commTest[ 1 ], commTest[ 2 ] );
mcbsp_util_fatal();
}
bsp_sync();
//test direct_get functionality
size_t commTest2[ 3 ];
commTest2[ 0 ] = commTest[ 0 ];
//get commTest[1] from right neighbour
bsp_direct_get( (bsp_pid() + 1) % bsp_nprocs(), &commTest, sizeof( size_t ), &(commTest2[ 1 ]), sizeof( size_t ) );
//get commTest[2] from left neighbour
bsp_direct_get( (bsp_pid() + bsp_nprocs() - 1) % bsp_nprocs(), &commTest, 2 * sizeof( size_t ), &(commTest2[ 2 ]), sizeof( size_t ) );
//now everything should equal bsp_pid
if( commTest2[ 0 ] != ((size_t)bsp_pid()) ||
commTest2[ 1 ] != ((size_t)bsp_pid()) ||
commTest2[ 2 ] != ((size_t)bsp_pid())
) {
fprintf( stderr, "FAILURE \t direct_get does not function properly! (%d: [%ld %ld %ld])\n", bsp_pid(), commTest2[ 0 ], commTest2[ 1 ], commTest2[ 2 ] );
mcbsp_util_fatal();
}
//now test single BSMP message
bsp_send( (bsp_pid() + 1) % bsp_nprocs(), NULL, &commTest, sizeof( size_t ) );
//check messages
const struct mcbsp_util_stack queue1 = data->queues[ (bsp_pid() + 1) % bsp_nprocs() ];
const size_t new_predicted_cap = predictCap( sizeof( struct mcbsp_message ) + sizeof( size_t ) );
predicted_cap = predicted_cap > new_predicted_cap ? predicted_cap : new_predicted_cap;
if( queue1.cap != predicted_cap || queue1.size != sizeof( struct mcbsp_message ) || queue1.top != sizeof( struct mcbsp_message ) + sizeof( size_t ) ) {
fprintf( stderr, "FAILURE \t bsp_send did not adapt the communication queue as expected!\n(cap = %ld, size = %ld, top = %ld; prediction was %ld, %ld, %ld)\n",
(size_t)queue1.cap, (size_t)queue1.size, (size_t)queue1.top,
(size_t)predicted_cap, (size_t)(sizeof( struct mcbsp_message )), (size_t)(sizeof( struct mcbsp_message ) + sizeof( size_t )) );
mcbsp_util_fatal();
}
const struct mcbsp_message request2 = *(struct mcbsp_message*) ((char*)queue1.array + queue1.top - sizeof( struct mcbsp_message ));
if( request2.destination != NULL ||
request2.length != sizeof( size_t ) || // assumes tagSize = 0
*(size_t *)queue1.array != ((size_t)bsp_pid()) ) {
fprintf( stderr, "FAILURE \t bsp_send did not push the expected communication request!\n(top = %ld, destination = %p, length = %ld, payload = %ld\n",
(size_t)queue1.top, request2.destination, (size_t)request2.length, *(size_t *)queue1.array );
mcbsp_util_fatal();
}
bsp_sync();
//inspect incoming BSMP queue (assuming tagSize = 0)
predicted_cap = predictCap( sizeof( size_t ) + sizeof( size_t ) );
if( data->bsmp.cap != predicted_cap || data->bsmp.top != sizeof( size_t ) + sizeof( size_t ) || data->bsmp.size != sizeof( size_t ) ) {
fprintf( stderr, "FAILURE \t BSMP queue after superstep with sends is not as expected!\n(cap = %ld, top = %ld, size = %ld; prediction was %ld, %ld, %ld)\n",
(size_t)data->bsmp.cap, (size_t)data->bsmp.top, (size_t)data->bsmp.size,
(size_t)predicted_cap, (size_t)(8 + sizeof( size_t )), (size_t)(data->bsmp.size) );
mcbsp_util_fatal();
}
if( *(size_t*)(data->bsmp.array) != (size_t)((bsp_pid() + bsp_nprocs() - 1) % bsp_nprocs()) ) {
fprintf( stderr, "FAILURE \t Value in BSMP queue is not correct!\n" );
mcbsp_util_fatal();
}
//inspect using primitives
MCBSP_NUMMSG_TYPE packets;
MCBSP_BYTESIZE_TYPE packetSize;
bsp_qsize( &packets, &packetSize );
if( packets != 1 || packetSize != sizeof( size_t ) ) {
fprintf( stderr, "FAILURE \t bsp_qsize does not function correctly!\n" );
mcbsp_util_fatal();
}
bsp_move( &commTest, sizeof( size_t ) );
if( commTest[ 0 ] != (size_t)(( bsp_pid() + bsp_nprocs() - 1 ) % bsp_nprocs()) ) {
fprintf( stderr, "FAILURE \t bsp_move does not function correctly!\n" );
mcbsp_util_fatal();
}
//check set_tagsize
MCBSP_BYTESIZE_TYPE tsz = sizeof( size_t );
bsp_set_tagsize( &tsz );
if( tsz != 0 ) {
fprintf( stderr, "FAILURE \t return value of bsp_set_tagsize is incorrect!\n" );
mcbsp_util_fatal();
}
bsp_sync();
//check set_tagsize
if( data->init->tagSize != sizeof( size_t ) ) {
fprintf( stderr, "FAILURE \t bsp_set_tagsize failed!\n" );
mcbsp_util_fatal();
}
commTest[ 0 ] = ((size_t)bsp_pid());
commTest[ 1 ] = 3;
commTest[ 2 ] = 8 + ((size_t)bsp_pid());
for( unsigned char i = 0; i < bsp_nprocs(); ++i ) {
bsp_send( i, commTest, &(commTest[1]), 2 * sizeof( size_t ) );
char * const test = (char*)(data->queues[ (size_t)i ].array) + data->queues[ (size_t)i ].top - sizeof( struct mcbsp_message ) - sizeof( size_t );
if( *(size_t*)test != *commTest ) {
fprintf( stderr, "FAILURE \t BSMP tag did not get pushed correctly (reads %ld instead of %ld)!\n", *(size_t*)test, *commTest );
mcbsp_util_fatal();
}
}
bsp_sync();
MCBSP_BYTESIZE_TYPE status;
size_t tag;
for( unsigned char i = 0; i < bsp_nprocs(); ++i ) {
bsp_get_tag( &status, &tag );
if( tag >= ((size_t)bsp_nprocs()) || status != 2 * sizeof( size_t ) ) {
fprintf( stderr, "FAILURE \t error in BSMP tag handling! (tag=%ld, status=%ld)\n", tag, (size_t)status );
mcbsp_util_fatal();
}
size_t *p_tag, *msg;
if( bsp_hpmove( (void**)&p_tag, (void**)&msg ) != 2 * sizeof( size_t ) ) {
fprintf( stderr, "FAILURE \t bsp_hpmove does not return correct payload length." );
}
if( msg[ 0 ] != 3 || *p_tag != tag ) {
fprintf( stderr, "FAILURE \t bsp_hpmove does not contain correct message (tag=%ld, payload = %ld) which should be (%ld, 3).\n", *p_tag, msg[ 0 ], tag );
mcbsp_util_fatal();
}
commTest[ tag ] = msg[ 1 ];
}
for( unsigned short int i = 0; i < bsp_nprocs(); ++i ) {
if( commTest[ i ] != (unsigned int)(8 + i) ) {
fprintf( stderr, "FAILURE \t error in bsp_tag / bsp_(hp)move combination!\n" );
mcbsp_util_fatal();
}
}
bsp_sync();
#ifdef MCBSP_ALLOW_MULTIPLE_REGS
//test multiple regs
double mreg[17];
bsp_push_reg( &(mreg[0]), 7*sizeof( double ) );
bsp_sync();
double mregs = 1.3;
bsp_put( (bsp_pid() + 1) % bsp_nprocs(), &mregs, &mreg, 6 * sizeof( double ), sizeof( double ) );
bsp_push_reg( &(mreg[0]), 17*sizeof( double ) );
bsp_sync();
bsp_push_reg( &(mreg[0]), 13*sizeof( double ) );
bsp_put( (bsp_pid() + 1) % bsp_nprocs(), &mregs, &mreg, 16 * sizeof( double ), sizeof( double ) );
bsp_sync();
if( mreg[ 6 ] != mreg[ 16 ] || mreg[ 6 ] != mregs ) {
fprintf( stderr, "FAILURE \t error in bsp_put + multiple bsp_push_reg calls (%f,%f,%f,...,%f,%f)\n", mreg[ 5 ], mreg[ 6 ], mreg[ 7 ], mreg[ 15 ], mreg[ 16 ] );
mcbsp_util_fatal();
}
bsp_pop_reg( &(mreg[0]) );
bsp_pop_reg( &(mreg[0]) );
bsp_sync();
bsp_put( (bsp_pid() + 1) % bsp_nprocs(), &mregs, &mreg, 2 * sizeof( double ), sizeof( double ) );
bsp_sync();
if( mreg[ 2 ] != mregs ) {
fprintf( stderr, "FAILURE \t error in bsp_put + multiple bsp_push_reg + multiple bsp_pop_reg calls\n" );
mcbsp_util_fatal();
}
#endif
bsp_end();
}
void EXT_MEM_TEXT
ebsp_send_up(const void* tag, const void* payload, int nbytes) {
coredata.read_queue_index = 1;
return bsp_send(-1, tag, payload, nbytes);
}
