void concurrent_vector_base::internal_grow( const size_type start, size_type finish, size_type element_size, internal_array_op1 init ) {
__TBB_ASSERT( start<finish, "start must be less than finish" );
size_t tmp = start;
do {
segment_index_t k_old = segment_index_of( tmp );
size_type base = segment_base(k_old);
size_t n = segment_size(k_old);
helper::extend_segment_if_necessary(*this,k_old);
segment_t& s = my_segment[k_old];
void* array = s.array;
if ( !array ) {
if ( base==tmp ) {
__TBB_ASSERT( !s.array, NULL );
array = NFS_Allocate( n, element_size, NULL );
ITT_NOTIFY( sync_releasing, &s.array );
s.array = array;
} else {
ITT_NOTIFY(sync_prepare, &s.array);
spin_wait_while_eq( s.array, (void*)0 );
ITT_NOTIFY(sync_acquired, &s.array);
array = s.array;
}
}
size_type j_begin = tmp-base;
size_type j_end = n > finish-base ? finish-base : n;
(*init)( (void*)((char*)array+element_size*j_begin), j_end-j_begin );
tmp = base+j_end;
} while( tmp<finish );
}
market& market::global_market ( unsigned max_num_workers, size_t stack_size ) {
global_market_mutex_type::scoped_lock lock( theMarketMutex );
market *m = theMarket;
if ( m ) {
++m->my_ref_count;
if ( m->my_stack_size < stack_size )
runtime_warning( "Newer master request for larger stack cannot be satisfied\n" );
}
else {
max_num_workers = max( governor::default_num_threads() - 1, max_num_workers );
// at least 1 worker is required to support starvation resistant tasks
if( max_num_workers==0 ) max_num_workers = 1;
// Create the global market instance
size_t size = sizeof(market);
#if __TBB_TASK_GROUP_CONTEXT
__TBB_ASSERT( __TBB_offsetof(market, my_workers) + sizeof(generic_scheduler*) == sizeof(market),
"my_workers must be the last data field of the market class");
size += sizeof(generic_scheduler*) * (max_num_workers - 1);
#endif /* __TBB_TASK_GROUP_CONTEXT */
__TBB_InitOnce::add_ref();
void* storage = NFS_Allocate(size, 1, NULL);
memset( storage, 0, size );
// Initialize and publish global market
m = new (storage) market( max_num_workers, stack_size );
theMarket = m;
}
return *m;
}
arena::arena ( market& m, unsigned max_num_workers ) {
__TBB_ASSERT( !my_guard, "improperly allocated arena?" );
__TBB_ASSERT( sizeof(slot[0]) % NFS_GetLineSize()==0, "arena::slot size not multiple of cache line size" );
__TBB_ASSERT( (uintptr_t)this % NFS_GetLineSize()==0, "arena misaligned" );
my_market = &m;
my_limit = 1;
// Two slots are mandatory: for the master, and for 1 worker (required to support starvation resistant tasks).
my_num_slots = max(2u, max_num_workers + 1);
my_max_num_workers = max_num_workers;
my_num_threads_active = 1; // accounts for the master
__TBB_ASSERT ( my_max_num_workers < my_num_slots, NULL );
// Construct mailboxes. Mark internal synchronization elements for the tools.
for( unsigned i = 0; i < my_num_slots; ++i ) {
__TBB_ASSERT( !slot[i].my_scheduler && !slot[i].task_pool, NULL );
ITT_SYNC_CREATE(slot + i, SyncType_Scheduler, SyncObj_WorkerTaskPool);
mailbox(i+1).construct();
ITT_SYNC_CREATE(&mailbox(i+1), SyncType_Scheduler, SyncObj_Mailbox);
#if __TBB_STATISTICS
slot[i].my_counters = new ( NFS_Allocate(sizeof(statistics_counters), 1, NULL) ) statistics_counters;
#endif /* __TBB_STATISTICS */
}
my_task_stream.initialize(my_num_slots);
ITT_SYNC_CREATE(&my_task_stream, SyncType_Scheduler, SyncObj_TaskStream);
my_mandatory_concurrency = false;
#if __TBB_TASK_GROUP_CONTEXT
my_master_default_ctx = NULL;
#endif
}
void* concurrent_vector_base::internal_push_back( size_type element_size, size_type& index ) {
__TBB_ASSERT( sizeof(my_early_size)==sizeof(reference_count), NULL );
//size_t tmp = __TBB_FetchAndIncrementWacquire(*(tbb::internal::reference_count*)&my_early_size);
size_t tmp = __TBB_FetchAndIncrementWacquire((tbb::internal::reference_count*)&my_early_size);
index = tmp;
segment_index_t k_old = segment_index_of( tmp );
size_type base = segment_base(k_old);
helper::extend_segment_if_necessary(*this,k_old);
segment_t& s = my_segment[k_old];
void* array = s.array;
if ( !array ) {
// FIXME - consider factoring this out and share with internal_grow_by
if ( base==tmp ) {
__TBB_ASSERT( !s.array, NULL );
size_t n = segment_size(k_old);
array = NFS_Allocate( n, element_size, NULL );
ITT_NOTIFY( sync_releasing, &s.array );
s.array = array;
} else {
ITT_NOTIFY(sync_prepare, &s.array);
spin_wait_while_eq( s.array, (void*)0 );
ITT_NOTIFY(sync_acquired, &s.array);
array = s.array;
}
}
size_type j_begin = tmp-base;
return (void*)((char*)array+element_size*j_begin);
}
void concurrent_vector_base::internal_assign( const concurrent_vector_base& src, size_type element_size, internal_array_op1 destroy, internal_array_op2 assign, internal_array_op2 copy ) {
size_type n = src.my_early_size;
while( my_early_size>n ) {
segment_index_t k = segment_index_of( my_early_size-1 );
size_type b=segment_base(k);
size_type new_end = b>=n ? b : n;
__TBB_ASSERT( my_early_size>new_end, NULL );
destroy( (char*)my_segment[k].array+element_size*(new_end-b), my_early_size-new_end );
my_early_size = new_end;
}
size_type dst_initialized_size = my_early_size;
my_early_size = n;
size_type b;
for( segment_index_t k=0; (b=segment_base(k))<n; ++k ) {
helper::extend_segment_if_necessary(*this,k);
size_t m = segment_size(k);
if ( !my_segment[k].array )
my_segment[k].array = NFS_Allocate( m, element_size, NULL );
if ( m>n-b ) m = n-b;
size_type a = 0;
if ( dst_initialized_size>b ) {
a = dst_initialized_size-b;
if ( a>m ) a = m;
assign( my_segment[k].array, src.my_segment[k].array, a );
m -= a;
a *= element_size;
}
if ( m>0 )
copy( (char*)my_segment[k].array+a, (char*)src.my_segment[k].array+a, m );
}
__TBB_ASSERT( src.my_early_size==n, "detected use of ConcurrentVector::operator= with right side that was concurrently modified" );
}
arena& arena::allocate_arena( market& m, unsigned max_num_workers ) {
__TBB_ASSERT( sizeof(base_type) + sizeof(arena_slot) == sizeof(arena), "All arena data fields must go to arena_base" );
__TBB_ASSERT( sizeof(base_type) % NFS_GetLineSize() == 0, "arena slots area misaligned: wrong padding" );
__TBB_ASSERT( sizeof(mail_outbox) == NFS_MaxLineSize, "Mailbox padding is wrong" );
size_t n = allocation_size(max_num_workers);
unsigned char* storage = (unsigned char*)NFS_Allocate( n, 1, NULL );
// Zero all slots to indicate that they are empty
memset( storage, 0, n );
return *new( storage + num_slots_to_reserve(max_num_workers) * sizeof(mail_outbox) ) arena(m, max_num_workers);
}
void concurrent_vector_base::internal_reserve( size_type n, size_type element_size, size_type max_size ) {
if ( n>max_size ) {
throw std::length_error("argument to ConcurrentVector::reserve exceeds ConcurrentVector::max_size()");
}
for( segment_index_t k = helper::find_segment_end(*this); segment_base(k)<n; ++k ) {
helper::extend_segment_if_necessary(*this,k);
size_t m = segment_size(k);
__TBB_ASSERT( !my_segment[k].array, "concurrent operation during reserve(...)?" );
my_segment[k].array = NFS_Allocate( m, element_size, NULL );
}
}
void concurrent_vector_base::helper::extend_segment( concurrent_vector_base& v ) {
const size_t pointers_per_long_segment = sizeof(void*)==4 ? 32 : 64;
segment_t* s = (segment_t*)NFS_Allocate( pointers_per_long_segment, sizeof(segment_t), NULL );
std::memset( static_cast<void*>(s), 0, pointers_per_long_segment*sizeof(segment_t) );
// If other threads are trying to set pointers in the short segment, wait for them to finish their
// assignments before we copy the short segment to the long segment.
atomic_backoff backoff;
while( !v.my_storage[0].array || !v.my_storage[1].array ) backoff.pause();
s[0] = v.my_storage[0];
s[1] = v.my_storage[1];
if( v.my_segment.compare_and_swap( s, v.my_storage )!=v.my_storage )
NFS_Free(s);
}
void concurrent_vector_base::internal_copy( const concurrent_vector_base& src, size_type element_size, internal_array_op2 copy ) {
size_type n = src.my_early_size;
my_early_size = n;
my_segment = my_storage;
if ( n ) {
size_type b;
for( segment_index_t k=0; (b=segment_base(k))<n; ++k ) {
helper::extend_segment_if_necessary(*this,k);
size_t m = segment_size(k);
__TBB_ASSERT( !my_segment[k].array, "concurrent operation during copy construction?" );
my_segment[k].array = NFS_Allocate( m, element_size, NULL );
if ( m>n-b ) m = n-b;
copy( my_segment[k].array, src.my_segment[k].array, m );
}
}
}
//------------------------------------------------------------------------
// Methods of affinity_partitioner_base_v3
//------------------------------------------------------------------------
void affinity_partitioner_base_v3::resize( unsigned factor ) {
// Check factor to avoid asking for number of workers while there might be no arena.
size_t new_size = factor ? factor*(governor::max_number_of_workers()+1) : 0;
if( new_size!=my_size ) {
if( my_array ) {
NFS_Free( my_array );
// Following two assignments must be done here for sake of exception safety.
my_array = NULL;
my_size = 0;
}
if( new_size ) {
my_array = static_cast<affinity_id*>(NFS_Allocate(new_size,sizeof(affinity_id), NULL ));
memset( my_array, 0, sizeof(affinity_id)*new_size );
my_size = new_size;
}
}
}
arena* arena::allocate_arena( unsigned number_of_slots, unsigned number_of_workers, stack_size_type stack_size ) {
__TBB_ASSERT( sizeof(ArenaPrefix) % NFS_GetLineSize()==0, "ArenaPrefix not multiple of cache line size" );
__TBB_ASSERT( sizeof(mail_outbox)==NFS_MaxLineSize, NULL );
__TBB_ASSERT( stack_size>0, NULL );
size_t n = sizeof(ArenaPrefix) + number_of_slots*(sizeof(mail_outbox)+sizeof(arena_slot));
unsigned char* storage = (unsigned char*)NFS_Allocate( n, 1, NULL );
// Zero all slots to indicate that they are empty
memset( storage, 0, n );
arena* a = (arena*)(storage + sizeof(ArenaPrefix)+ number_of_slots*(sizeof(mail_outbox)));
__TBB_ASSERT( sizeof(a->slot[0]) % NFS_GetLineSize()==0, "arena::slot size not multiple of cache line size" );
__TBB_ASSERT( (uintptr_t)a % NFS_GetLineSize()==0, NULL );
new( &a->prefix() ) ArenaPrefix( number_of_slots, number_of_workers );
// Allocate the worker_list
WorkerDescriptor * w = new WorkerDescriptor[number_of_workers];
memset( w, 0, sizeof(WorkerDescriptor)*(number_of_workers));
a->prefix().worker_list = w;
// Construct mailboxes.
for( unsigned j=1; j<=number_of_slots; ++j )
a->mailbox(j).construct();
a->prefix().stack_size = stack_size;
size_t k;
// Mark each internal sync element for the tools
for( k=0; k<number_of_workers; ++k ) {
ITT_SYNC_CREATE(a->slot + k, SyncType_Scheduler, SyncObj_WorkerTaskPool);
ITT_SYNC_CREATE(&w[k].scheduler, SyncType_Scheduler, SyncObj_WorkerLifeCycleMgmt);
ITT_SYNC_CREATE(&a->mailbox(k+1), SyncType_Scheduler, SyncObj_Mailbox);
}
for( ; k<number_of_slots; ++k ) {
ITT_SYNC_CREATE(a->slot + k, SyncType_Scheduler, SyncObj_MasterTaskPool);
ITT_SYNC_CREATE(&a->mailbox(k+1), SyncType_Scheduler, SyncObj_Mailbox);
}
return a;
}