scheduler_base(std::size_t num_threads)
: topology_(get_topology())
, affinity_data_(num_threads)
#if defined(HPX_HAVE_THREAD_MANAGER_IDLE_BACKOFF)
, wait_count_(0)
#endif
{}
///////////////////////////////////////////////////////////////////////////
// Return the number of the NUMA node the current thread is running on
std::size_t get_numa_node_number()
{
bool numa_sensitive = false;
std::size_t thread_num =
threadmanager_base::get_worker_thread_num(&numa_sensitive);
return get_topology().get_numa_node_number(
get_thread_manager().get_pu_num(thread_num));
}
scheduler_base(std::size_t num_threads)
: topology_(get_topology())
, affinity_data_(num_threads)
#if defined(HPX_THREAD_BACKOFF_ON_IDLE)
, wait_count_(0)
, waiting_(false)
#endif
{}
std::size_t thread_pool<Scheduler>::init(std::size_t num_threads,
policies::init_affinity_data const& data)
{
topology const& topology_ = get_topology();
std::size_t cores_used = sched_.Scheduler::init(data, topology_);
resize(used_processing_units_, threads::hardware_concurrency());
for (std::size_t i = 0; i != num_threads; ++i)
used_processing_units_ |= sched_.Scheduler::get_pu_mask(topology_, i);
return cores_used;
}
std::string get_non_unique_key_messages(const stk::mesh::BulkData& bulkData, const std::vector<stk::mesh::EntityKeyProc> &badKeyProcs)
{
std::ostringstream os;
for(const stk::mesh::EntityKeyProc& keyProc : badKeyProcs)
{
stk::mesh::Entity entity = bulkData.get_entity(keyProc.first);
os << "[" << bulkData.parallel_rank() << "] Key " << keyProc.first <<
get_topology(bulkData.bucket(entity).topology()) << "is also present (inappropriately) on processor " <<
keyProc.second << "." << std::endl;
}
return os.str();
}
bool initialize_topology(void) {
/* this is done when execution daemon starts */
if (logical_used_topology == NULL) {
if (get_topology(&logical_used_topology, &logical_used_topology_length)) {
return true;
}
}
return false;
}
scheduler_base(std::size_t num_threads,
scheduler_mode mode = nothing_special)
: topology_(get_topology())
, affinity_data_(num_threads)
, mode_(mode)
#if defined(HPX_HAVE_THREAD_MANAGER_IDLE_BACKOFF)
, wait_count_(0)
#endif
{
states_.resize(num_threads);
for (std::size_t i = 0; i != num_threads; ++i)
states_[i].store(state_initialized);
}
/****** sge_binding/account_job() **********************************************
* NAME
* account_job() -- Accounts core binding from a job on host global topology.
*
* SYNOPSIS
* bool account_job(char* job_topology)
*
* FUNCTION
* Accounts core binding from a job on host global topology.
*
* INPUTS
* char* job_topology - Topology used from core binding.
*
* RESULT
* bool - true when successful otherwise false
*
* NOTES
* MT-NOTE: account_job() is not MT safe
*
* SEE ALSO
* ???/???
*******************************************************************************/
bool account_job(const char* job_topology)
{
if (logical_used_topology_length == 0 || logical_used_topology == NULL) {
/* initialize without any usage */
if (!get_topology(&logical_used_topology,
&logical_used_topology_length))
return false;
}
return account_job_on_topology(&logical_used_topology,
strlen(logical_used_topology),
job_topology, strlen(job_topology));
}
/****** sge_binding/getExecdTopologyInUse() ************************************
* NAME
* getExecdTopologyInUse() -- Creates a string which represents the used topology.
*
* SYNOPSIS
* bool getExecdTopologyInUse(char** topology)
*
* FUNCTION
*
* Checks all jobs (with going through active jobs directories) and their
* usage of the topology (binding). Afterwards global "logical_used_topology"
* string is up to date (which is also updated when a job ends and starts) and
* a copy is made available for the caller.
*
* Note: The memory is allocated within this function and
* has to be freed from the caller afterwards.
* INPUTS
* char** topology - out: the current topology in use by jobs
*
* RESULT
* bool - true if the "topology in use" string could be created
*
* NOTES
* MT-NOTE: getExecdTopologyInUse() is not MT safe
*******************************************************************************/
bool get_execd_topology_in_use(char** topology)
{
bool retval = false;
/* topology must be a NULL pointer */
if ((*topology) != NULL) {
return false;
}
if (logical_used_topology_length == 0 || logical_used_topology == NULL) {
/* initialize without any usage */
get_topology(&logical_used_topology, &logical_used_topology_length);
}
if (logical_used_topology_length > 0) {
/* copy the string */
(*topology) = sge_strdup(NULL, logical_used_topology);
retval = true;
}
return retval;
}
///////////////////////////////////////////////////////////////////////////
// Return the number of the NUMA node the current thread is running on
std::size_t get_numa_node_number()
{
std::size_t thread_num = hpx::get_worker_thread_num();
return get_topology().get_numa_node_number(
get_thread_manager().get_pu_num(thread_num));
}
bool thread_pool<Scheduler>::run(boost::unique_lock<boost::mutex>& l,
std::size_t num_threads)
{
HPX_ASSERT(l.owns_lock());
LTM_(info) //-V128
<< "thread_pool::run: " << pool_name_
<< " number of processing units available: " //-V128
<< threads::hardware_concurrency();
LTM_(info) //-V128
<< "thread_pool::run: " << pool_name_
<< " creating " << num_threads << " OS thread(s)"; //-V128
if (0 == num_threads) {
HPX_THROW_EXCEPTION(bad_parameter,
"thread_pool::run", "number of threads is zero");
}
#if defined(HPX_HAVE_THREAD_CUMULATIVE_COUNTS) && \
defined(HPX_HAVE_THREAD_IDLE_RATES)
// scale timestamps to nanoseconds
boost::uint64_t base_timestamp = util::hardware::timestamp();
boost::uint64_t base_time = util::high_resolution_clock::now();
boost::uint64_t curr_timestamp = util::hardware::timestamp();
boost::uint64_t curr_time = util::high_resolution_clock::now();
while ((curr_time - base_time) <= 100000)
{
curr_timestamp = util::hardware::timestamp();
curr_time = util::high_resolution_clock::now();
}
if (curr_timestamp - base_timestamp != 0)
{
timestamp_scale_ = double(curr_time - base_time) /
double(curr_timestamp - base_timestamp);
}
LTM_(info)
<< "thread_pool::run: " << pool_name_
<< " timestamp_scale: " << timestamp_scale_; //-V128
#endif
if (!threads_.empty() || sched_.has_reached_state(state_running))
return true; // do nothing if already running
executed_threads_.resize(num_threads);
executed_thread_phases_.resize(num_threads);
tfunc_times_.resize(num_threads);
exec_times_.resize(num_threads);
try {
HPX_ASSERT(startup_.get() == 0);
startup_.reset(
new boost::barrier(static_cast<unsigned>(num_threads+1))
);
// run threads and wait for initialization to complete
sched_.set_all_states(state_running);
topology const& topology_ = get_topology();
std::size_t thread_num = num_threads;
while (thread_num-- != 0) {
threads::mask_cref_type mask =
sched_.Scheduler::get_pu_mask(topology_, thread_num);
LTM_(info) //-V128
<< "thread_pool::run: " << pool_name_
<< " create OS thread " << thread_num //-V128
<< ": will run on processing units within this mask: "
#if !defined(HPX_WITH_MORE_THAN_64_THREADS) || \
(defined(HPX_HAVE_MAX_CPU_COUNT) && HPX_HAVE_MAX_CPU_COUNT <= 64)
<< std::hex << "0x" << mask;
#else
<< "0b" << mask;
#endif
// create a new thread
threads_.push_back(new boost::thread(
util::bind(&thread_pool::thread_func, this, thread_num,
boost::ref(topology_), boost::ref(*startup_))
));
// set the new threads affinity (on Windows systems)
if (any(mask))
{
error_code ec(lightweight);
topology_.set_thread_affinity_mask(threads_.back(), mask, ec);
if (ec)
{
LTM_(warning) //-V128
<< "thread_pool::run: " << pool_name_
<< " setting thread affinity on OS thread " //-V128
<< thread_num << " failed with: "
<< ec.get_message();
}
}
else
{
LTM_(debug) //-V128
<< "thread_pool::run: " << pool_name_
<< " setting thread affinity on OS thread " //-V128
<< thread_num << " was explicitly disabled.";
}
}
// the main thread needs to have a unique thread_num
init_tss(num_threads);
startup_->wait();
}
/****** sge_binding/binding_explicit_check_and_account() ***********************
* NAME
* binding_explicit_check_and_account() -- Checks if a job can be bound.
*
* SYNOPSIS
* bool binding_explicit_check_and_account(const int* list_of_sockets, const
* int samount, const int** list_of_cores, const int score, char**
* topo_used_by_job, int* topo_used_by_job_length)
*
* FUNCTION
* Checks if the job can bind to the given by the <socket>,<core> pairs.
* If so these cores are marked as used and true is returned. Also an
* topology string is returned where all cores consumed by the job are
* marked with smaller case letters.
*
* INPUTS
* const int* list_of_sockets - List of sockets to be used
* const int samount - Size of list_of_sockets
* const int** list_of_cores - List of cores (on sockets) to be used
* const int score - Size of list_of_cores
*
* OUTPUTS
* char** topo_used_by_job - Topology with resources job consumes marked.
* int* topo_used_by_job_length - Topology string length.
*
* RESULT
* bool - True if the job can be bound to the topology, false if not.
*
* NOTES
* MT-NOTE: binding_explicit_check_and_account() is MT safe
*
* SEE ALSO
* ???/???
*******************************************************************************/
bool binding_explicit_check_and_account(const int* list_of_sockets, const int samount,
const int* list_of_cores, const int score, char** topo_used_by_job,
int* topo_used_by_job_length)
{
int i;
/* position of <socket>,<core> in topology string */
int pos;
/* status if accounting was possible */
bool possible = true;
/* input parameter validation */
if (samount != score || samount <= 0 || list_of_sockets == NULL
|| list_of_cores == NULL) {
return false;
}
/* check if the topology which is used already is accessable */
if (logical_used_topology == NULL) {
/* we have no topology string at the moment (should be initialized before) */
if (!get_topology(&logical_used_topology, &logical_used_topology_length)) {
/* couldn't even get the topology string */
return false;
}
}
/* create output string */
get_topology(topo_used_by_job, topo_used_by_job_length);
/* go through the <socket>,<core> pair list */
for (i = 0; i < samount; i++) {
/* get position in topology string */
if ((pos = get_position_in_topology(list_of_sockets[i], list_of_cores[i],
logical_used_topology, logical_used_topology_length)) < 0) {
/* the <socket>,<core> does not exist */
possible = false;
break;
}
/* check if this core is available (DG TODO introduce threads) */
if (logical_used_topology[pos] == 'C') {
/* do temporarily account it */
(*topo_used_by_job)[pos] = 'c';
/* thread binding: account threads here */
account_all_threads_after_core(topo_used_by_job, pos);
} else {
/* core not usable -> early abort */
possible = false;
break;
}
}
/* do accounting if all cores can be used */
if (possible) {
if (account_job_on_topology(&logical_used_topology, logical_used_topology_length,
*topo_used_by_job, *topo_used_by_job_length) == false) {
possible = false;
}
}
/* free memory when unsuccessful */
if (possible == false) {
sge_free(topo_used_by_job);
*topo_used_by_job_length = 0;
}
return possible;
}
/****** sge_binding/get_striding_first_socket_first_core_and_account() ********
* NAME
* get_striding_first_socket_first_core_and_account() -- Checks if and where
* striding would fit.
*
* SYNOPSIS
* bool getStridingFirstSocketFirstCore(const int amount, const int
* stepsize, int* first_socket, int* first_core)
*
* FUNCTION
* This operating system independent function checks (depending on
* the underlaying topology string and the topology string which
* reflects already execution units in use) if it is possible to
* bind the job in a striding manner to cores on the host.
*
* This function requires the topology string and the string with the
* topology currently in use.
*
* INPUTS
* const int amount - Amount of cores to allocate.
* const int stepsize - Distance of the cores to allocate.
* const int start_at_socket - First socket to begin the search with (usually at 0).
* const int start_at_core - First core to begin the search with (usually at 0).
* int* first_socket - out: First socket when striding is possible (return value).
* int* first_core - out: First core when striding is possible (return value).
*
* RESULT
* bool - if true striding is possible at <first_socket, first_core>
*
* NOTES
* MT-NOTE: getStridingFirstSocketFirstCore() is not MT safe
*
* SEE ALSO
* ???/???
*******************************************************************************/
bool get_striding_first_socket_first_core_and_account(const int amount, const int stepsize,
const int start_at_socket, const int start_at_core, const bool automatic,
int* first_socket, int* first_core, char** accounted_topology,
int* accounted_topology_length)
{
/* return value: if it is possible to fit the request on the host */
bool possible = false;
/* position in topology string */
int i = 0;
/* socket and core counter in order to find the first core and socket */
int sc = -1;
int cc = -1;
/* these core and socket counters are added later on .. */
int found_cores = 0;
int found_sockets = 0; /* first socket is given implicitely */
/* temp topology string where accounting is done on */
char* tmp_topo_busy;
/* initialize socket and core where the striding will fit */
*first_socket = 0;
*first_core = 0;
if (start_at_socket < 0 || start_at_core < 0) {
/* wrong input parameter */
return false;
}
if (logical_used_topology == NULL) {
/* we have no topology string at the moment (should be initialized before) */
if (!get_topology(&logical_used_topology, &logical_used_topology_length)) {
/* couldn't even get the topology string */
return false;
}
}
/* temporary accounting string -> account on this and
when eventually successful then copy this string back
to global topo_busy string */
tmp_topo_busy = (char *) calloc(logical_used_topology_length + 1, sizeof(char));
memcpy(tmp_topo_busy, logical_used_topology, logical_used_topology_length*sizeof(char));
/* we have to go to the first position given by the arguments
(start_at_socket and start_at_core) */
for (i = 0; i < logical_used_topology_length; i++) {
if (logical_used_topology[i] == 'C' || logical_used_topology[i] == 'c') {
/* found core -> update core counter */
cc++;
} else if (logical_used_topology[i] == 'S' || logical_used_topology[i] == 's') {
/* found socket -> update socket counter */
sc++;
/* we're changing socket -> no core found on this one yet */
cc = -1;
} else if (logical_used_topology[i] == '\0') {
/* we couldn't find start socket start string */
possible = false;
sge_free(&tmp_topo_busy);
return possible;
}
if (sc == start_at_socket && cc == start_at_core) {
/* we found our starting point (we remember 'i' for next loop!) */
break;
}
}
/* check if we found the socket and core we want to start searching */
if (sc != start_at_socket || cc != start_at_core) {
/* could't find the start socket and start core */
sge_free(&tmp_topo_busy);
return false;
}
/* check each position of the topology string */
/* we reuse 'i' from last loop -> this is the position where we begin */
for (; i < logical_used_topology_length && logical_used_topology[i] != '\0'; i++) {
/* this could be optimized (with increasing i in case if it is not
possible) */
if (is_starting_point(logical_used_topology, logical_used_topology_length, i, amount, stepsize,
&tmp_topo_busy)) {
/* we can do striding with this as starting point */
possible = true;
/* update place where we can begin */
*first_socket = start_at_socket + found_sockets;
*first_core = start_at_core + found_cores;
/* return the accounted topology */
create_topology_used_per_job(accounted_topology, accounted_topology_length,
logical_used_topology, tmp_topo_busy, logical_used_topology_length);
/* finally do execution host wide accounting */
/* DG TODO mutex */
memcpy(logical_used_topology, tmp_topo_busy, logical_used_topology_length*sizeof(char));
break;
} else {
/* else retry and update socket and core number to start with */
if (logical_used_topology[i] == 'C' || logical_used_topology[i] == 'c') {
/* jumping over a core */
found_cores++;
/* a core is a valid starting point for binding in non-automatic case */
/* if we have a fixed start socket and a start core we do not retry
it with the next core available (when introducing T's this have to
be added there too) */
if (automatic == false) {
possible = false;
break;
}
} else if (logical_used_topology[i] == 'S' || logical_used_topology[i] == 's') {
/* jumping over a socket */
found_sockets++;
/* we are at core 0 on the new socket */
found_cores = 0;
}
/* at the moment we are not interested in threads or anything else */
}
} /* end go through the whole topology string */
sge_free(&tmp_topo_busy);
return possible;
}
resource_manager::resource_manager()
: next_cookie_(0),
punits_(get_os_thread_count()),
topology_(get_topology())
{}
