static #ifdef KMP_INLINE_SUBR __forceinline #endif struct private_common * __kmp_threadprivate_find_task_common( struct common_table *tbl, int gtid, void *pc_addr ) { struct private_common *tn; #ifdef KMP_TASK_COMMON_DEBUG KC_TRACE( 10, ( "__kmp_threadprivate_find_task_common: thread#%d, called with address %p\n", gtid, pc_addr ) ); dump_list(); #endif for (tn = tbl->data[ KMP_HASH(pc_addr) ]; tn; tn = tn->next) { if (tn->gbl_addr == pc_addr) { #ifdef KMP_TASK_COMMON_DEBUG KC_TRACE( 10, ( "__kmp_threadprivate_find_task_common: thread#%d, found node %p on list\n", gtid, pc_addr ) ); #endif return tn; } } return 0; }
/* Call all destructors for threadprivate data belonging to this thread */ void __kmp_common_destroy_gtid( int gtid ) { struct private_common *tn; struct shared_common *d_tn; KC_TRACE( 10, ("__kmp_common_destroy_gtid: T#%d called\n", gtid ) ); if( (__kmp_foreign_tp) ? (! KMP_INITIAL_GTID (gtid)) : (! KMP_UBER_GTID (gtid)) ) { if( TCR_4(__kmp_init_common) ) { #if KMP_THREADPRIVATE_TLS for(d_tn = kmpc_threadprivate_d_table_data_head_local; d_tn != NULL; d_tn = d_tn->next) { // call destructor if (d_tn->dt.dtor) d_tn->dt.dtor(NULL); } #else /* Cannot do this here since not all threads have destroyed their data */ /* TCW_4(__kmp_init_common, FALSE); */ for (tn = __kmp_threads[ gtid ]->th.th_pri_head; tn; tn = tn->link) { d_tn = __kmp_find_shared_task_common( &__kmp_threadprivate_d_table, gtid, tn->gbl_addr ); KMP_DEBUG_ASSERT( d_tn ); if (d_tn->is_vec) { if (d_tn->dt.dtorv != 0) { (void) (*d_tn->dt.dtorv) (tn->par_addr, d_tn->vec_len); } if (d_tn->obj_init != 0) { (void) (*d_tn->dt.dtorv) (d_tn->obj_init, d_tn->vec_len); } } else { if (d_tn->dt.dtor != 0) { (void) (*d_tn->dt.dtor) (tn->par_addr); } if (d_tn->obj_init != 0) { (void) (*d_tn->dt.dtor) (d_tn->obj_init); } } } #endif KC_TRACE( 30, ("__kmp_common_destroy_gtid: T#%d threadprivate destructors complete\n", gtid ) ); } } }
/*! @ingroup THREADPRIVATE @param loc source location information @param data pointer to data being privatized @param ctor pointer to constructor function for data @param cctor pointer to copy constructor function for data @param dtor pointer to destructor function for data Register constructors and destructors for thread private data. This function is called when executing in parallel, when we know the thread id. */ void __kmpc_threadprivate_register(ident_t *loc, void *data, kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor) { struct shared_common *d_tn, **lnk_tn; KC_TRACE( 10, ("__kmpc_threadprivate_register: called\n" ) ); #ifdef USE_CHECKS_COMMON /* copy constructor must be zero for current code gen (Nov 2002 - jph) */ KMP_ASSERT( cctor == 0); #endif /* USE_CHECKS_COMMON */ /* Only the global data table exists. */ d_tn = __kmp_find_shared_task_common( &__kmp_threadprivate_d_table, -1, data ); if (d_tn == 0) { d_tn = (struct shared_common *) __kmp_allocate( sizeof( struct shared_common ) ); d_tn->gbl_addr = data; d_tn->ct.ctor = ctor; d_tn->cct.cctor = cctor; d_tn->dt.dtor = dtor; /* d_tn->is_vec = FALSE; // AC: commented out because __kmp_allocate zeroes the memory d_tn->vec_len = 0L; d_tn->obj_init = 0; d_tn->pod_init = 0; */ lnk_tn = &(__kmp_threadprivate_d_table.data[ KMP_HASH(data) ]); d_tn->next = *lnk_tn; *lnk_tn = d_tn; } }
static void dump_list( void ) { int p, q; for (p = 0; p < __kmp_all_nth; ++p) { if( !__kmp_threads[p] ) continue; for (q = 0; q < KMP_HASH_TABLE_SIZE; ++q) { if (__kmp_threads[ p ]->th.th_pri_common->data[ q ]) { struct private_common *tn; KC_TRACE( 10, ( "\tdump_list: gtid:%d addresses\n", p ) ); for (tn = __kmp_threads[ p ]->th.th_pri_common->data[ q ]; tn; tn = tn->next) { KC_TRACE( 10, ( "\tdump_list: THREADPRIVATE: Serial %p -> Parallel %p\n", tn->gbl_addr, tn->par_addr ) ); } } } } }
void * __kmpc_threadprivate(ident_t *loc, kmp_int32 global_tid, void *data, size_t size) { void *ret; struct private_common *tn; KC_TRACE( 10, ("__kmpc_threadprivate: T#%d called\n", global_tid ) ); #ifdef USE_CHECKS_COMMON if (! __kmp_init_serial) KMP_FATAL( RTLNotInitialized ); #endif /* USE_CHECKS_COMMON */ if ( ! __kmp_threads[global_tid] -> th.th_root -> r.r_active && ! __kmp_foreign_tp ) { /* The parallel address will NEVER overlap with the data_address */ /* dkp: 3rd arg to kmp_threadprivate_insert_private_data() is the data_address; use data_address = data */ KC_TRACE( 20, ("__kmpc_threadprivate: T#%d inserting private data\n", global_tid ) ); kmp_threadprivate_insert_private_data( global_tid, data, data, size ); ret = data; } else { KC_TRACE( 50, ("__kmpc_threadprivate: T#%d try to find private data at address %p\n", global_tid, data ) ); tn = __kmp_threadprivate_find_task_common( __kmp_threads[ global_tid ]->th.th_pri_common, global_tid, data ); if ( tn ) { KC_TRACE( 20, ("__kmpc_threadprivate: T#%d found data\n", global_tid ) ); #ifdef USE_CHECKS_COMMON if ((size_t) size > tn->cmn_size) { KC_TRACE( 10, ( "THREADPRIVATE: %p (%" KMP_UINTPTR_SPEC " ,%" KMP_UINTPTR_SPEC ")\n", data, size, tn->cmn_size ) ); KMP_FATAL( TPCommonBlocksInconsist ); } #endif /* USE_CHECKS_COMMON */ } else { /* The parallel address will NEVER overlap with the data_address */ /* dkp: 3rd arg to kmp_threadprivate_insert() is the data_address; use data_address = data */ KC_TRACE( 20, ("__kmpc_threadprivate: T#%d inserting data\n", global_tid ) ); tn = kmp_threadprivate_insert( global_tid, data, data, size ); } ret = tn->par_addr; } KC_TRACE( 10, ("__kmpc_threadprivate: T#%d exiting; return value = %p\n", global_tid, ret ) ); return ret; }
static #ifdef KMP_INLINE_SUBR __forceinline #endif struct shared_common * __kmp_find_shared_task_common( struct shared_table *tbl, int gtid, void *pc_addr ) { struct shared_common *tn; for (tn = tbl->data[ KMP_HASH(pc_addr) ]; tn; tn = tn->next) { if (tn->gbl_addr == pc_addr) { #ifdef KMP_TASK_COMMON_DEBUG KC_TRACE( 10, ( "__kmp_find_shared_task_common: thread#%d, found node %p on list\n", gtid, pc_addr ) ); #endif return tn; } } return 0; }
/*! @ingroup THREADPRIVATE @param loc source location information @param global_tid global thread number @param data pointer to data to privatize @param size size of data to privatize @param cache pointer to cache @return pointer to private storage Allocate private storage for threadprivate data. */ void * __kmpc_threadprivate_cached( ident_t * loc, kmp_int32 global_tid, // gtid. void * data, // Pointer to original global variable. size_t size, // Size of original global variable. void *** cache ) { KC_TRACE( 10, ("__kmpc_threadprivate_cached: T#%d called with cache: %p, address: %p, size: %" KMP_SIZE_T_SPEC "\n", global_tid, *cache, data, size ) ); if ( TCR_PTR(*cache) == 0) { __kmp_acquire_lock( & __kmp_global_lock, global_tid ); if ( TCR_PTR(*cache) == 0) { __kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock); __kmp_tp_cached = 1; __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock); void ** my_cache; KMP_ITT_IGNORE( my_cache = (void**) __kmp_allocate(sizeof( void * ) * __kmp_tp_capacity + sizeof ( kmp_cached_addr_t )); );
/*! @ingroup CANCELLATION @param loc_ref location of the original task directive @param gtid Global thread ID of encountering thread @param cncl_kind Cancellation kind (parallel, for, sections, taskgroup) @return returns true if the cancellation request has been activated and the execution thread needs to proceed to the end of the canceled region. Request cancellation of the binding OpenMP region. */ kmp_int32 __kmpc_cancel(ident_t *loc_ref, kmp_int32 gtid, kmp_int32 cncl_kind) { kmp_info_t *this_thr = __kmp_threads[gtid]; KC_TRACE(10, ("__kmpc_cancel: T#%d request %d OMP_CANCELLATION=%d\n", gtid, cncl_kind, __kmp_omp_cancellation)); KMP_DEBUG_ASSERT(cncl_kind != cancel_noreq); KMP_DEBUG_ASSERT(cncl_kind == cancel_parallel || cncl_kind == cancel_loop || cncl_kind == cancel_sections || cncl_kind == cancel_taskgroup); KMP_DEBUG_ASSERT(__kmp_get_gtid() == gtid); if (__kmp_omp_cancellation) { switch (cncl_kind) { case cancel_parallel: case cancel_loop: case cancel_sections: // cancellation requests for parallel and worksharing constructs // are handled through the team structure { kmp_team_t *this_team = this_thr->th.th_team; KMP_DEBUG_ASSERT(this_team); kmp_int32 old = cancel_noreq; this_team->t.t_cancel_request.compare_exchange_strong(old, cncl_kind); if (old == cancel_noreq || old == cncl_kind) { // we do not have a cancellation request in this team or we do have // one that matches the current request -> cancel #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.ompt_callback_cancel) { ompt_data_t *task_data; __ompt_get_task_info_internal(0, NULL, &task_data, NULL, NULL, NULL); ompt_cancel_flag_t type = ompt_cancel_parallel; if (cncl_kind == cancel_parallel) type = ompt_cancel_parallel; else if (cncl_kind == cancel_loop) type = ompt_cancel_loop; else if (cncl_kind == cancel_sections) type = ompt_cancel_sections; ompt_callbacks.ompt_callback(ompt_callback_cancel)( task_data, type | ompt_cancel_activated, OMPT_GET_RETURN_ADDRESS(0)); } #endif return 1 /* true */; } break; } case cancel_taskgroup: // cancellation requests for a task group // are handled through the taskgroup structure { kmp_taskdata_t *task; kmp_taskgroup_t *taskgroup; task = this_thr->th.th_current_task; KMP_DEBUG_ASSERT(task); taskgroup = task->td_taskgroup; if (taskgroup) { kmp_int32 old = cancel_noreq; taskgroup->cancel_request.compare_exchange_strong(old, cncl_kind); if (old == cancel_noreq || old == cncl_kind) { // we do not have a cancellation request in this taskgroup or we do // have one that matches the current request -> cancel #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.ompt_callback_cancel) { ompt_data_t *task_data; __ompt_get_task_info_internal(0, NULL, &task_data, NULL, NULL, NULL); ompt_callbacks.ompt_callback(ompt_callback_cancel)( task_data, ompt_cancel_taskgroup | ompt_cancel_activated, OMPT_GET_RETURN_ADDRESS(0)); } #endif return 1 /* true */; } } else { // TODO: what needs to happen here? // the specification disallows cancellation w/o taskgroups // so we might do anything here, let's abort for now KMP_ASSERT(0 /* false */); } } break; default: KMP_ASSERT(0 /* false */); } } // ICV OMP_CANCELLATION=false, so we ignored this cancel request KMP_DEBUG_ASSERT(!__kmp_omp_cancellation); return 0 /* false */; }
/*! @ingroup CANCELLATION @param loc_ref location of the original task directive @param gtid Global thread ID of encountering thread @param cncl_kind Cancellation kind (parallel, for, sections, taskgroup) @return returns true if a matching cancellation request has been flagged in the RTL and the encountering thread has to cancel.. Cancellation point for the encountering thread. */ kmp_int32 __kmpc_cancellationpoint(ident_t *loc_ref, kmp_int32 gtid, kmp_int32 cncl_kind) { kmp_info_t *this_thr = __kmp_threads[gtid]; KC_TRACE(10, ("__kmpc_cancellationpoint: T#%d request %d OMP_CANCELLATION=%d\n", gtid, cncl_kind, __kmp_omp_cancellation)); KMP_DEBUG_ASSERT(cncl_kind != cancel_noreq); KMP_DEBUG_ASSERT(cncl_kind == cancel_parallel || cncl_kind == cancel_loop || cncl_kind == cancel_sections || cncl_kind == cancel_taskgroup); KMP_DEBUG_ASSERT(__kmp_get_gtid() == gtid); if (__kmp_omp_cancellation) { switch (cncl_kind) { case cancel_parallel: case cancel_loop: case cancel_sections: // cancellation requests for parallel and worksharing constructs // are handled through the team structure { kmp_team_t *this_team = this_thr->th.th_team; KMP_DEBUG_ASSERT(this_team); if (this_team->t.t_cancel_request) { if (cncl_kind == this_team->t.t_cancel_request) { // the request in the team structure matches the type of // cancellation point so we can cancel #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.ompt_callback_cancel) { ompt_data_t *task_data; __ompt_get_task_info_internal(0, NULL, &task_data, NULL, NULL, NULL); ompt_cancel_flag_t type = ompt_cancel_parallel; if (cncl_kind == cancel_parallel) type = ompt_cancel_parallel; else if (cncl_kind == cancel_loop) type = ompt_cancel_loop; else if (cncl_kind == cancel_sections) type = ompt_cancel_sections; ompt_callbacks.ompt_callback(ompt_callback_cancel)( task_data, type | ompt_cancel_detected, OMPT_GET_RETURN_ADDRESS(0)); } #endif return 1 /* true */; } KMP_ASSERT(0 /* false */); } else { // we do not have a cancellation request pending, so we just // ignore this cancellation point return 0; } break; } case cancel_taskgroup: // cancellation requests for a task group // are handled through the taskgroup structure { kmp_taskdata_t *task; kmp_taskgroup_t *taskgroup; task = this_thr->th.th_current_task; KMP_DEBUG_ASSERT(task); taskgroup = task->td_taskgroup; if (taskgroup) { // return the current status of cancellation for the taskgroup #if OMPT_SUPPORT && OMPT_OPTIONAL if (ompt_enabled.ompt_callback_cancel && !!taskgroup->cancel_request) { ompt_data_t *task_data; __ompt_get_task_info_internal(0, NULL, &task_data, NULL, NULL, NULL); ompt_callbacks.ompt_callback(ompt_callback_cancel)( task_data, ompt_cancel_taskgroup | ompt_cancel_detected, OMPT_GET_RETURN_ADDRESS(0)); } #endif return !!taskgroup->cancel_request; } else { // if a cancellation point is encountered by a task that does not // belong to a taskgroup, it is OK to ignore it return 0 /* false */; } } default: KMP_ASSERT(0 /* false */); } } // ICV OMP_CANCELLATION=false, so we ignore the cancellation point KMP_DEBUG_ASSERT(!__kmp_omp_cancellation); return 0 /* false */; }
/*! @ingroup THREADPRIVATE @param loc source location information @param data pointer to data being privatized @param ctor pointer to constructor function for data @param cctor pointer to copy constructor function for data @param dtor pointer to destructor function for data Register constructors and destructors for thread private data. This function is called when executing in parallel, when we know the thread id. */ void __kmpc_threadprivate_register(ident_t *loc, void *data, kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor) { struct shared_common *d_tn, **lnk_tn; KC_TRACE( 10, ("__kmpc_threadprivate_register: called\n" ) ); #ifdef USE_CHECKS_COMMON /* copy constructor must be zero for current code gen (Nov 2002 - jph) */ KMP_ASSERT( cctor == 0); #endif /* USE_CHECKS_COMMON */ #if KMP_THREADPRIVATE_TLS /* Check if unique. We use here only data[0], as we need to maintain a uniqe ordering between each of the entries on the list. This is because each thread maintain a private variable pointing to the entries it has already initialized. When a worker starts a parallel region, it will check if his private pointer points to the most recent entry. If not, the thread will call the constructors for all entries between the top and the ones already processed. */ for (d_tn = __kmp_threadprivate_d_table.data[0]; d_tn; d_tn = d_tn->next) { if (d_tn->gbl_addr == data) { // nothing to be done, already here return; } } // not found, create one d_tn = (struct shared_common *) __kmp_allocate( sizeof( struct shared_common ) ); d_tn->gbl_addr = data; d_tn->ct.ctor = ctor; d_tn->cct.cctor = cctor; d_tn->dt.dtor = dtor; // use only one list (0th, arbitrary) lnk_tn = &(__kmp_threadprivate_d_table.data[0]); // set new element at head of list d_tn->next = __kmp_threadprivate_d_table.data[0]; // Make sure that if a thread see the new element, then it must // see the new next pointer value. KMP_MB(); __kmp_threadprivate_d_table.data[0] = d_tn; return; #endif /* Only the global data table exists. */ d_tn = __kmp_find_shared_task_common( &__kmp_threadprivate_d_table, -1, data ); if (d_tn == 0) { d_tn = (struct shared_common *) __kmp_allocate( sizeof( struct shared_common ) ); d_tn->gbl_addr = data; d_tn->ct.ctor = ctor; d_tn->cct.cctor = cctor; d_tn->dt.dtor = dtor; /* d_tn->is_vec = FALSE; // AC: commented out because __kmp_allocate zeroes the memory d_tn->vec_len = 0L; d_tn->obj_init = 0; d_tn->pod_init = 0; */ lnk_tn = &(__kmp_threadprivate_d_table.data[ KMP_HASH(data) ]); d_tn->next = *lnk_tn; *lnk_tn = d_tn; } }
struct private_common * kmp_threadprivate_insert( int gtid, void *pc_addr, void *data_addr, size_t pc_size ) { struct private_common *tn, **tt; struct shared_common *d_tn; /* +++++++++ START OF CRITICAL SECTION +++++++++ */ __kmp_acquire_lock( & __kmp_global_lock, gtid ); tn = (struct private_common *) __kmp_allocate( sizeof (struct private_common) ); tn->gbl_addr = pc_addr; d_tn = __kmp_find_shared_task_common( &__kmp_threadprivate_d_table, gtid, pc_addr ); /* Only the MASTER data table exists. */ if (d_tn != 0) { /* This threadprivate variable has already been seen. */ if ( d_tn->pod_init == 0 && d_tn->obj_init == 0 ) { d_tn->cmn_size = pc_size; if (d_tn->is_vec) { if (d_tn->ct.ctorv != 0) { /* Construct from scratch so no prototype exists */ d_tn->obj_init = 0; } else if (d_tn->cct.cctorv != 0) { /* Now data initialize the prototype since it was previously registered */ d_tn->obj_init = (void *) __kmp_allocate( d_tn->cmn_size ); (void) (*d_tn->cct.cctorv) (d_tn->obj_init, pc_addr, d_tn->vec_len); } else { d_tn->pod_init = __kmp_init_common_data( data_addr, d_tn->cmn_size ); } } else { if (d_tn->ct.ctor != 0) { /* Construct from scratch so no prototype exists */ d_tn->obj_init = 0; } else if (d_tn->cct.cctor != 0) { /* Now data initialize the prototype since it was previously registered */ d_tn->obj_init = (void *) __kmp_allocate( d_tn->cmn_size ); (void) (*d_tn->cct.cctor) (d_tn->obj_init, pc_addr); } else { d_tn->pod_init = __kmp_init_common_data( data_addr, d_tn->cmn_size ); } } } } else { struct shared_common **lnk_tn; d_tn = (struct shared_common *) __kmp_allocate( sizeof( struct shared_common ) ); d_tn->gbl_addr = pc_addr; d_tn->cmn_size = pc_size; d_tn->pod_init = __kmp_init_common_data( data_addr, pc_size ); /* d_tn->obj_init = 0; // AC: commented out because __kmp_allocate zeroes the memory d_tn->ct.ctor = 0; d_tn->cct.cctor = 0; d_tn->dt.dtor = 0; d_tn->is_vec = FALSE; d_tn->vec_len = 0L; */ lnk_tn = &(__kmp_threadprivate_d_table.data[ KMP_HASH(pc_addr) ]); d_tn->next = *lnk_tn; *lnk_tn = d_tn; } tn->cmn_size = d_tn->cmn_size; if ( (__kmp_foreign_tp) ? (KMP_INITIAL_GTID (gtid)) : (KMP_UBER_GTID (gtid)) ) { tn->par_addr = (void *) pc_addr; } else { tn->par_addr = (void *) __kmp_allocate( tn->cmn_size ); } __kmp_release_lock( & __kmp_global_lock, gtid ); /* +++++++++ END OF CRITICAL SECTION +++++++++ */ #ifdef USE_CHECKS_COMMON if (pc_size > d_tn->cmn_size) { KC_TRACE( 10, ( "__kmp_threadprivate_insert: THREADPRIVATE: %p (%" KMP_UINTPTR_SPEC " ,%" KMP_UINTPTR_SPEC ")\n", pc_addr, pc_size, d_tn->cmn_size ) ); KMP_FATAL( TPCommonBlocksInconsist ); } #endif /* USE_CHECKS_COMMON */ tt = &(__kmp_threads[ gtid ]->th.th_pri_common->data[ KMP_HASH(pc_addr) ]); #ifdef KMP_TASK_COMMON_DEBUG if (*tt != 0) { KC_TRACE( 10, ( "__kmp_threadprivate_insert: WARNING! thread#%d: collision on %p\n", gtid, pc_addr ) ); } #endif tn->next = *tt; *tt = tn; #ifdef KMP_TASK_COMMON_DEBUG KC_TRACE( 10, ( "__kmp_threadprivate_insert: thread#%d, inserted node %p on list\n", gtid, pc_addr ) ); dump_list( ); #endif /* Link the node into a simple list */ tn->link = __kmp_threads[ gtid ]->th.th_pri_head; __kmp_threads[ gtid ]->th.th_pri_head = tn; #ifdef BUILD_TV __kmp_tv_threadprivate_store( __kmp_threads[ gtid ], tn->gbl_addr, tn->par_addr ); #endif if( (__kmp_foreign_tp) ? (KMP_INITIAL_GTID (gtid)) : (KMP_UBER_GTID (gtid)) ) return tn; /* * if C++ object with copy constructor, use it; * else if C++ object with constructor, use it for the non-master copies only; * else use pod_init and memcpy * * C++ constructors need to be called once for each non-master thread on allocate * C++ copy constructors need to be called once for each thread on allocate */ /* * C++ object with constructors/destructors; * don't call constructors for master thread though */ if (d_tn->is_vec) { if ( d_tn->ct.ctorv != 0) { (void) (*d_tn->ct.ctorv) (tn->par_addr, d_tn->vec_len); } else if (d_tn->cct.cctorv != 0) { (void) (*d_tn->cct.cctorv) (tn->par_addr, d_tn->obj_init, d_tn->vec_len); } else if (tn->par_addr != tn->gbl_addr) { __kmp_copy_common_data( tn->par_addr, d_tn->pod_init ); } } else { if ( d_tn->ct.ctor != 0 ) { (void) (*d_tn->ct.ctor) (tn->par_addr); } else if (d_tn->cct.cctor != 0) { (void) (*d_tn->cct.cctor) (tn->par_addr, d_tn->obj_init); } else if (tn->par_addr != tn->gbl_addr) { __kmp_copy_common_data( tn->par_addr, d_tn->pod_init ); } } /* !BUILD_OPENMP_C if (tn->par_addr != tn->gbl_addr) __kmp_copy_common_data( tn->par_addr, d_tn->pod_init ); */ return tn; }