scheduler_base(std::size_t num_threads) : topology_(get_topology()) , affinity_data_(num_threads) #if defined(HPX_HAVE_THREAD_MANAGER_IDLE_BACKOFF) , wait_count_(0) #endif {}
/////////////////////////////////////////////////////////////////////////// // Return the number of the NUMA node the current thread is running on std::size_t get_numa_node_number() { bool numa_sensitive = false; std::size_t thread_num = threadmanager_base::get_worker_thread_num(&numa_sensitive); return get_topology().get_numa_node_number( get_thread_manager().get_pu_num(thread_num)); }
scheduler_base(std::size_t num_threads) : topology_(get_topology()) , affinity_data_(num_threads) #if defined(HPX_THREAD_BACKOFF_ON_IDLE) , wait_count_(0) , waiting_(false) #endif {}
std::size_t thread_pool<Scheduler>::init(std::size_t num_threads, policies::init_affinity_data const& data) { topology const& topology_ = get_topology(); std::size_t cores_used = sched_.Scheduler::init(data, topology_); resize(used_processing_units_, threads::hardware_concurrency()); for (std::size_t i = 0; i != num_threads; ++i) used_processing_units_ |= sched_.Scheduler::get_pu_mask(topology_, i); return cores_used; }
std::string get_non_unique_key_messages(const stk::mesh::BulkData& bulkData, const std::vector<stk::mesh::EntityKeyProc> &badKeyProcs) { std::ostringstream os; for(const stk::mesh::EntityKeyProc& keyProc : badKeyProcs) { stk::mesh::Entity entity = bulkData.get_entity(keyProc.first); os << "[" << bulkData.parallel_rank() << "] Key " << keyProc.first << get_topology(bulkData.bucket(entity).topology()) << "is also present (inappropriately) on processor " << keyProc.second << "." << std::endl; } return os.str(); }
bool initialize_topology(void) { /* this is done when execution daemon starts */ if (logical_used_topology == NULL) { if (get_topology(&logical_used_topology, &logical_used_topology_length)) { return true; } } return false; }
scheduler_base(std::size_t num_threads, scheduler_mode mode = nothing_special) : topology_(get_topology()) , affinity_data_(num_threads) , mode_(mode) #if defined(HPX_HAVE_THREAD_MANAGER_IDLE_BACKOFF) , wait_count_(0) #endif { states_.resize(num_threads); for (std::size_t i = 0; i != num_threads; ++i) states_[i].store(state_initialized); }
/****** sge_binding/account_job() ********************************************** * NAME * account_job() -- Accounts core binding from a job on host global topology. * * SYNOPSIS * bool account_job(char* job_topology) * * FUNCTION * Accounts core binding from a job on host global topology. * * INPUTS * char* job_topology - Topology used from core binding. * * RESULT * bool - true when successful otherwise false * * NOTES * MT-NOTE: account_job() is not MT safe * * SEE ALSO * ???/??? *******************************************************************************/ bool account_job(const char* job_topology) { if (logical_used_topology_length == 0 || logical_used_topology == NULL) { /* initialize without any usage */ if (!get_topology(&logical_used_topology, &logical_used_topology_length)) return false; } return account_job_on_topology(&logical_used_topology, strlen(logical_used_topology), job_topology, strlen(job_topology)); }
/****** sge_binding/getExecdTopologyInUse() ************************************ * NAME * getExecdTopologyInUse() -- Creates a string which represents the used topology. * * SYNOPSIS * bool getExecdTopologyInUse(char** topology) * * FUNCTION * * Checks all jobs (with going through active jobs directories) and their * usage of the topology (binding). Afterwards global "logical_used_topology" * string is up to date (which is also updated when a job ends and starts) and * a copy is made available for the caller. * * Note: The memory is allocated within this function and * has to be freed from the caller afterwards. * INPUTS * char** topology - out: the current topology in use by jobs * * RESULT * bool - true if the "topology in use" string could be created * * NOTES * MT-NOTE: getExecdTopologyInUse() is not MT safe *******************************************************************************/ bool get_execd_topology_in_use(char** topology) { bool retval = false; /* topology must be a NULL pointer */ if ((*topology) != NULL) { return false; } if (logical_used_topology_length == 0 || logical_used_topology == NULL) { /* initialize without any usage */ get_topology(&logical_used_topology, &logical_used_topology_length); } if (logical_used_topology_length > 0) { /* copy the string */ (*topology) = sge_strdup(NULL, logical_used_topology); retval = true; } return retval; }
/////////////////////////////////////////////////////////////////////////// // Return the number of the NUMA node the current thread is running on std::size_t get_numa_node_number() { std::size_t thread_num = hpx::get_worker_thread_num(); return get_topology().get_numa_node_number( get_thread_manager().get_pu_num(thread_num)); }
bool thread_pool<Scheduler>::run(boost::unique_lock<boost::mutex>& l, std::size_t num_threads) { HPX_ASSERT(l.owns_lock()); LTM_(info) //-V128 << "thread_pool::run: " << pool_name_ << " number of processing units available: " //-V128 << threads::hardware_concurrency(); LTM_(info) //-V128 << "thread_pool::run: " << pool_name_ << " creating " << num_threads << " OS thread(s)"; //-V128 if (0 == num_threads) { HPX_THROW_EXCEPTION(bad_parameter, "thread_pool::run", "number of threads is zero"); } #if defined(HPX_HAVE_THREAD_CUMULATIVE_COUNTS) && \ defined(HPX_HAVE_THREAD_IDLE_RATES) // scale timestamps to nanoseconds boost::uint64_t base_timestamp = util::hardware::timestamp(); boost::uint64_t base_time = util::high_resolution_clock::now(); boost::uint64_t curr_timestamp = util::hardware::timestamp(); boost::uint64_t curr_time = util::high_resolution_clock::now(); while ((curr_time - base_time) <= 100000) { curr_timestamp = util::hardware::timestamp(); curr_time = util::high_resolution_clock::now(); } if (curr_timestamp - base_timestamp != 0) { timestamp_scale_ = double(curr_time - base_time) / double(curr_timestamp - base_timestamp); } LTM_(info) << "thread_pool::run: " << pool_name_ << " timestamp_scale: " << timestamp_scale_; //-V128 #endif if (!threads_.empty() || sched_.has_reached_state(state_running)) return true; // do nothing if already running executed_threads_.resize(num_threads); executed_thread_phases_.resize(num_threads); tfunc_times_.resize(num_threads); exec_times_.resize(num_threads); try { HPX_ASSERT(startup_.get() == 0); startup_.reset( new boost::barrier(static_cast<unsigned>(num_threads+1)) ); // run threads and wait for initialization to complete sched_.set_all_states(state_running); topology const& topology_ = get_topology(); std::size_t thread_num = num_threads; while (thread_num-- != 0) { threads::mask_cref_type mask = sched_.Scheduler::get_pu_mask(topology_, thread_num); LTM_(info) //-V128 << "thread_pool::run: " << pool_name_ << " create OS thread " << thread_num //-V128 << ": will run on processing units within this mask: " #if !defined(HPX_WITH_MORE_THAN_64_THREADS) || \ (defined(HPX_HAVE_MAX_CPU_COUNT) && HPX_HAVE_MAX_CPU_COUNT <= 64) << std::hex << "0x" << mask; #else << "0b" << mask; #endif // create a new thread threads_.push_back(new boost::thread( util::bind(&thread_pool::thread_func, this, thread_num, boost::ref(topology_), boost::ref(*startup_)) )); // set the new threads affinity (on Windows systems) if (any(mask)) { error_code ec(lightweight); topology_.set_thread_affinity_mask(threads_.back(), mask, ec); if (ec) { LTM_(warning) //-V128 << "thread_pool::run: " << pool_name_ << " setting thread affinity on OS thread " //-V128 << thread_num << " failed with: " << ec.get_message(); } } else { LTM_(debug) //-V128 << "thread_pool::run: " << pool_name_ << " setting thread affinity on OS thread " //-V128 << thread_num << " was explicitly disabled."; } } // the main thread needs to have a unique thread_num init_tss(num_threads); startup_->wait(); }
/****** sge_binding/binding_explicit_check_and_account() *********************** * NAME * binding_explicit_check_and_account() -- Checks if a job can be bound. * * SYNOPSIS * bool binding_explicit_check_and_account(const int* list_of_sockets, const * int samount, const int** list_of_cores, const int score, char** * topo_used_by_job, int* topo_used_by_job_length) * * FUNCTION * Checks if the job can bind to the given by the <socket>,<core> pairs. * If so these cores are marked as used and true is returned. Also an * topology string is returned where all cores consumed by the job are * marked with smaller case letters. * * INPUTS * const int* list_of_sockets - List of sockets to be used * const int samount - Size of list_of_sockets * const int** list_of_cores - List of cores (on sockets) to be used * const int score - Size of list_of_cores * * OUTPUTS * char** topo_used_by_job - Topology with resources job consumes marked. * int* topo_used_by_job_length - Topology string length. * * RESULT * bool - True if the job can be bound to the topology, false if not. * * NOTES * MT-NOTE: binding_explicit_check_and_account() is MT safe * * SEE ALSO * ???/??? *******************************************************************************/ bool binding_explicit_check_and_account(const int* list_of_sockets, const int samount, const int* list_of_cores, const int score, char** topo_used_by_job, int* topo_used_by_job_length) { int i; /* position of <socket>,<core> in topology string */ int pos; /* status if accounting was possible */ bool possible = true; /* input parameter validation */ if (samount != score || samount <= 0 || list_of_sockets == NULL || list_of_cores == NULL) { return false; } /* check if the topology which is used already is accessable */ if (logical_used_topology == NULL) { /* we have no topology string at the moment (should be initialized before) */ if (!get_topology(&logical_used_topology, &logical_used_topology_length)) { /* couldn't even get the topology string */ return false; } } /* create output string */ get_topology(topo_used_by_job, topo_used_by_job_length); /* go through the <socket>,<core> pair list */ for (i = 0; i < samount; i++) { /* get position in topology string */ if ((pos = get_position_in_topology(list_of_sockets[i], list_of_cores[i], logical_used_topology, logical_used_topology_length)) < 0) { /* the <socket>,<core> does not exist */ possible = false; break; } /* check if this core is available (DG TODO introduce threads) */ if (logical_used_topology[pos] == 'C') { /* do temporarily account it */ (*topo_used_by_job)[pos] = 'c'; /* thread binding: account threads here */ account_all_threads_after_core(topo_used_by_job, pos); } else { /* core not usable -> early abort */ possible = false; break; } } /* do accounting if all cores can be used */ if (possible) { if (account_job_on_topology(&logical_used_topology, logical_used_topology_length, *topo_used_by_job, *topo_used_by_job_length) == false) { possible = false; } } /* free memory when unsuccessful */ if (possible == false) { sge_free(topo_used_by_job); *topo_used_by_job_length = 0; } return possible; }
/****** sge_binding/get_striding_first_socket_first_core_and_account() ******** * NAME * get_striding_first_socket_first_core_and_account() -- Checks if and where * striding would fit. * * SYNOPSIS * bool getStridingFirstSocketFirstCore(const int amount, const int * stepsize, int* first_socket, int* first_core) * * FUNCTION * This operating system independent function checks (depending on * the underlaying topology string and the topology string which * reflects already execution units in use) if it is possible to * bind the job in a striding manner to cores on the host. * * This function requires the topology string and the string with the * topology currently in use. * * INPUTS * const int amount - Amount of cores to allocate. * const int stepsize - Distance of the cores to allocate. * const int start_at_socket - First socket to begin the search with (usually at 0). * const int start_at_core - First core to begin the search with (usually at 0). * int* first_socket - out: First socket when striding is possible (return value). * int* first_core - out: First core when striding is possible (return value). * * RESULT * bool - if true striding is possible at <first_socket, first_core> * * NOTES * MT-NOTE: getStridingFirstSocketFirstCore() is not MT safe * * SEE ALSO * ???/??? *******************************************************************************/ bool get_striding_first_socket_first_core_and_account(const int amount, const int stepsize, const int start_at_socket, const int start_at_core, const bool automatic, int* first_socket, int* first_core, char** accounted_topology, int* accounted_topology_length) { /* return value: if it is possible to fit the request on the host */ bool possible = false; /* position in topology string */ int i = 0; /* socket and core counter in order to find the first core and socket */ int sc = -1; int cc = -1; /* these core and socket counters are added later on .. */ int found_cores = 0; int found_sockets = 0; /* first socket is given implicitely */ /* temp topology string where accounting is done on */ char* tmp_topo_busy; /* initialize socket and core where the striding will fit */ *first_socket = 0; *first_core = 0; if (start_at_socket < 0 || start_at_core < 0) { /* wrong input parameter */ return false; } if (logical_used_topology == NULL) { /* we have no topology string at the moment (should be initialized before) */ if (!get_topology(&logical_used_topology, &logical_used_topology_length)) { /* couldn't even get the topology string */ return false; } } /* temporary accounting string -> account on this and when eventually successful then copy this string back to global topo_busy string */ tmp_topo_busy = (char *) calloc(logical_used_topology_length + 1, sizeof(char)); memcpy(tmp_topo_busy, logical_used_topology, logical_used_topology_length*sizeof(char)); /* we have to go to the first position given by the arguments (start_at_socket and start_at_core) */ for (i = 0; i < logical_used_topology_length; i++) { if (logical_used_topology[i] == 'C' || logical_used_topology[i] == 'c') { /* found core -> update core counter */ cc++; } else if (logical_used_topology[i] == 'S' || logical_used_topology[i] == 's') { /* found socket -> update socket counter */ sc++; /* we're changing socket -> no core found on this one yet */ cc = -1; } else if (logical_used_topology[i] == '\0') { /* we couldn't find start socket start string */ possible = false; sge_free(&tmp_topo_busy); return possible; } if (sc == start_at_socket && cc == start_at_core) { /* we found our starting point (we remember 'i' for next loop!) */ break; } } /* check if we found the socket and core we want to start searching */ if (sc != start_at_socket || cc != start_at_core) { /* could't find the start socket and start core */ sge_free(&tmp_topo_busy); return false; } /* check each position of the topology string */ /* we reuse 'i' from last loop -> this is the position where we begin */ for (; i < logical_used_topology_length && logical_used_topology[i] != '\0'; i++) { /* this could be optimized (with increasing i in case if it is not possible) */ if (is_starting_point(logical_used_topology, logical_used_topology_length, i, amount, stepsize, &tmp_topo_busy)) { /* we can do striding with this as starting point */ possible = true; /* update place where we can begin */ *first_socket = start_at_socket + found_sockets; *first_core = start_at_core + found_cores; /* return the accounted topology */ create_topology_used_per_job(accounted_topology, accounted_topology_length, logical_used_topology, tmp_topo_busy, logical_used_topology_length); /* finally do execution host wide accounting */ /* DG TODO mutex */ memcpy(logical_used_topology, tmp_topo_busy, logical_used_topology_length*sizeof(char)); break; } else { /* else retry and update socket and core number to start with */ if (logical_used_topology[i] == 'C' || logical_used_topology[i] == 'c') { /* jumping over a core */ found_cores++; /* a core is a valid starting point for binding in non-automatic case */ /* if we have a fixed start socket and a start core we do not retry it with the next core available (when introducing T's this have to be added there too) */ if (automatic == false) { possible = false; break; } } else if (logical_used_topology[i] == 'S' || logical_used_topology[i] == 's') { /* jumping over a socket */ found_sockets++; /* we are at core 0 on the new socket */ found_cores = 0; } /* at the moment we are not interested in threads or anything else */ } } /* end go through the whole topology string */ sge_free(&tmp_topo_busy); return possible; }
resource_manager::resource_manager() : next_cookie_(0), punits_(get_os_thread_count()), topology_(get_topology()) {}