int main(int argc,
char *argv[])
{
aligned_t ret;
qthread_init(2);
CHECK_VERBOSE();
assert(qthread_num_shepherds() == 2);
iprintf("now to fork to shepherd 0...\n");
qthread_fork_to(checkres, (void *)0, &ret, 0);
qthread_readFF(&ret, &ret);
iprintf("success in forking to shepherd 0!\n");
iprintf("now to fork to shepherd 1...\n");
qthread_fork_to(checkres, (void *)1, &ret, 1);
qthread_readFF(&ret, &ret);
iprintf("success in forking to shepherd 1!\n");
iprintf("now to fork the migrant...\n");
qthread_fork(migrant, NULL, &ret);
iprintf("success in forking migrant!\n");
qthread_readFF(&ret, &ret);
iprintf("migrant returned successfully!\n");
return 0;
}
// Test that writeFF waits for empty var to be filled, writes, and leaves full.
// Requires that only one worker is running. Basically does:
// 1: empty var
// 1: fork(writeFF)
// 1: yields
// 2: starts runnning
// 2: hits writeFF, and yields since var is empty
// 1: writeEF
// 1: hits readFF on forked task and yield
// 2: running again, finishes writeFF, task returns
// 1: readFF competes, finishes
static void testWriteFFWaits(void)
{
aligned_t ret;
concurrent_t=45;
qthread_empty(&concurrent_t);
assert(qthread_num_workers() == 1);
iprintf("1: Forking writeFF wrapper\n");
qthread_fork_to(writeFF_wrapper, NULL, &ret, qthread_shep());
iprintf("1: Forked, now yielding to 2\n");
qthread_yield();
iprintf("1: Back from yield\n");
// verify that writeFF has not completed
assert(qthread_feb_status(&concurrent_t) == 0);
assert(concurrent_t != 55);
iprintf("1: Writing EF\n");
qthread_writeEF_const(&concurrent_t, 35);
// wait for writeFF wrapper to complete
qthread_readFF(NULL, &ret);
// veify that writeFF completed and that FEB is full
iprintf("1: concurrent_t=%d\n", concurrent_t);
assert(qthread_feb_status(&concurrent_t) == 1);
assert(concurrent_t == 55);
}
void QthreadExec::exec_all( Qthread & , QthreadExecFunctionPointer func , const void * arg )
{
verify_is_process("QthreadExec::exec_all(...)",true);
s_active_function = func ;
s_active_function_arg = arg ;
// Need to query which shepherd this main 'process' is running...
const int main_shep = qthread_shep();
for ( int jshep = 0 , iwork = 0 ; jshep < s_number_shepherds ; ++jshep ) {
for ( int i = jshep != main_shep ? 0 : 1 ; i < s_number_workers_per_shepherd ; ++i , ++iwork ) {
// Unit tests hang with this call:
//
// qthread_fork_to_local_priority( driver_exec_all , NULL , NULL , jshep );
//
qthread_fork_to( driver_exec_all , NULL , NULL , jshep );
}}
driver_exec_all( NULL );
s_active_function = 0 ;
s_active_function_arg = 0 ;
}
void accalt_tasklet_creation_to(void(*thread_func)(void *), void *arg, ACCALT_tasklet *new_ult, int dest) {
#ifdef ARGOBOTS
ABT_pool pool;
ABT_xstream_get_main_pools(main_team->team[dest], 1, &pool);
ABT_task_create(pool, thread_func, arg, new_ult);
#endif
#ifdef MASSIVETHREADS
accalt_ult_creation(thread_func, arg, new_ult);
#endif
#ifdef QTHREADS
qthread_fork_to((void *) thread_func, arg, new_ult, dest);
#endif
}
void QthreadExec::resize_worker_scratch( const int reduce_size , const int shared_size )
{
const int exec_all_reduce_alloc = align_alloc( reduce_size );
const int shepherd_scan_alloc = align_alloc( 8 );
const int shepherd_shared_end = exec_all_reduce_alloc + shepherd_scan_alloc + align_alloc( shared_size );
if ( s_worker_reduce_end < exec_all_reduce_alloc ||
s_worker_shared_end < shepherd_shared_end ) {
// Clear current worker memory before allocating new worker memory
clear_workers();
// Increase the buffers to an aligned allocation
s_worker_reduce_end = exec_all_reduce_alloc ;
s_worker_shared_begin = exec_all_reduce_alloc + shepherd_scan_alloc ;
s_worker_shared_end = shepherd_shared_end ;
// Need to query which shepherd this main 'process' is running...
// Have each worker resize its memory for proper first-touch
for ( int jshep = 0 ; jshep < s_number_shepherds ; ++jshep ) {
for ( int i = jshep ? 0 : 1 ; i < s_number_workers_per_shepherd ; ++i ) {
// Unit tests hang with this call:
//
// qthread_fork_to_local_priority( driver_resize_workers , NULL , NULL , jshep );
//
qthread_fork_to( driver_resize_worker_scratch , NULL , NULL , jshep );
}}
driver_resize_worker_scratch( NULL );
// Verify all workers allocated
bool ok = true ;
for ( int iwork = 0 ; ok && iwork < s_number_workers ; ++iwork ) { ok = 0 != s_exec[iwork] ; }
if ( ! ok ) {
std::ostringstream msg ;
msg << "Kokkos::Impl::QthreadExec::resize : FAILED for workers {" ;
for ( int iwork = 0 ; iwork < s_number_workers ; ++iwork ) {
if ( 0 == s_exec[iwork] ) { msg << " " << ( s_number_workers - ( iwork + 1 ) ); }
}
msg << " }" ;
Kokkos::Impl::throw_runtime_exception( msg.str() );
}
}
}
void QthreadExec::exec_all( Qthread & , QthreadExecFunctionPointer func , const void * arg )
{
verify_is_process("QthreadExec::exec_all(...)",true);
/*
fprintf( stdout , "QthreadExec::exec_all\n");
fflush(stdout);
*/
s_active_function = func ;
s_active_function_arg = arg ;
// Need to query which shepherd this main 'process' is running...
const int main_shep = qthread_shep();
#if 1
for ( int jshep = 0 , iwork = 0 ; jshep < s_number_shepherds ; ++jshep ) {
for ( int i = jshep != main_shep ? 0 : 1 ; i < s_number_workers_per_shepherd ; ++i , ++iwork ) {
qthread_fork_to( driver_exec_all , NULL , NULL , jshep );
}}
#else
// If this function is used before the 'qthread.task_policy' unit test
// the 'qthread.task_policy' unit test fails with a seg-fault within libqthread.so.
for ( int jshep = 0 ; jshep < s_number_shepherds ; ++jshep ) {
const int num_clone = jshep != main_shep ? s_number_workers_per_shepherd : s_number_workers_per_shepherd - 1 ;
if ( num_clone ) {
const int ret = qthread_fork_clones_to_local_priority
( driver_exec_all /* function */
, NULL /* function data block */
, NULL /* pointer to return value feb */
, jshep /* shepherd number */
, num_clone - 1 /* number of instances - 1 */
);
assert(ret == QTHREAD_SUCCESS);
}
}
#endif
driver_exec_all( NULL );
s_active_function = 0 ;
s_active_function_arg = 0 ;
}
void QthreadExec::resize_worker_scratch( const int reduce_size , const int shared_size )
{
const int exec_all_reduce_alloc = align_alloc( reduce_size );
const int shepherd_scan_alloc = align_alloc( 8 );
const int shepherd_shared_end = exec_all_reduce_alloc + shepherd_scan_alloc + align_alloc( shared_size );
if ( s_worker_reduce_end < exec_all_reduce_alloc ||
s_worker_shared_end < shepherd_shared_end ) {
/*
fprintf( stdout , "QthreadExec::resize\n");
fflush(stdout);
*/
// Clear current worker memory before allocating new worker memory
clear_workers();
// Increase the buffers to an aligned allocation
s_worker_reduce_end = exec_all_reduce_alloc ;
s_worker_shared_begin = exec_all_reduce_alloc + shepherd_scan_alloc ;
s_worker_shared_end = shepherd_shared_end ;
// Need to query which shepherd this main 'process' is running...
const int main_shep = qthread_shep();
// Have each worker resize its memory for proper first-touch
#if 1
for ( int jshep = 0 ; jshep < s_number_shepherds ; ++jshep ) {
for ( int i = jshep != main_shep ? 0 : 1 ; i < s_number_workers_per_shepherd ; ++i ) {
qthread_fork_to( driver_resize_worker_scratch , NULL , NULL , jshep );
}}
#else
// If this function is used before the 'qthread.task_policy' unit test
// the 'qthread.task_policy' unit test fails with a seg-fault within libqthread.so.
for ( int jshep = 0 ; jshep < s_number_shepherds ; ++jshep ) {
const int num_clone = jshep != main_shep ? s_number_workers_per_shepherd : s_number_workers_per_shepherd - 1 ;
if ( num_clone ) {
const int ret = qthread_fork_clones_to_local_priority
( driver_resize_worker_scratch /* function */
, NULL /* function data block */
, NULL /* pointer to return value feb */
, jshep /* shepherd number */
, num_clone - 1 /* number of instances - 1 */
);
assert(ret == QTHREAD_SUCCESS);
}
}
#endif
driver_resize_worker_scratch( NULL );
// Verify all workers allocated
bool ok = true ;
for ( int iwork = 0 ; ok && iwork < s_number_workers ; ++iwork ) { ok = 0 != s_exec[iwork] ; }
if ( ! ok ) {
std::ostringstream msg ;
msg << "Kokkos::Impl::QthreadExec::resize : FAILED for workers {" ;
for ( int iwork = 0 ; iwork < s_number_workers ; ++iwork ) {
if ( 0 == s_exec[iwork] ) { msg << " " << ( s_number_workers - ( iwork + 1 ) ); }
}
msg << " }" ;
Kokkos::Impl::throw_runtime_exception( msg.str() );
}
}
}
