static int rmaps_lama_find_nth_subtree_match_core(hwloc_topology_t hwloc_topo,
hwloc_obj_t parent_obj,
int nth,
int *num_found,
hwloc_obj_type_t hwloc_key,
int depth,
hwloc_obj_t *cur_child)
{
unsigned i;
bool found = false;
#if 0
{
char str[128];
hwloc_obj_snprintf(str, sizeof(str), hwloc_topo, parent_obj, "#", 0);
printf("--> Checking -- %-20s \t -- \t %2d of %2d\n", str, nth, *num_found);
}
#endif
/*
* Check if the keys match
*/
if( hwloc_key == parent_obj->type ) {
if( HWLOC_OBJ_CACHE == parent_obj->type &&
depth == (int)parent_obj->attr->cache.depth ) {
*num_found += 1;
found = true;
} else {
*num_found += 1;
found = true;
}
}
/*
* Basecase:
* If we have found the correct item, return
*/
if( nth == *num_found ) {
*cur_child = parent_obj;
return ORTE_SUCCESS;
}
/*
* Do no go any deeper in the tree than we have to
*/
if( !found ) {
for(i = 0; i < parent_obj->arity; ++i ) {
rmaps_lama_find_nth_subtree_match_core(hwloc_topo,
parent_obj->children[i],
nth,
num_found,
hwloc_key,
depth,
cur_child);
if( nth == *num_found ) {
return ORTE_SUCCESS;
}
}
}
return ORTE_SUCCESS;
}
hwloc_obj_t * rmaps_lama_find_nth_subtree_match(hwloc_topology_t hwloc_topo,
hwloc_obj_t parent_obj,
int nth,
rmaps_lama_level_type_t lama_key)
{
int ret;
hwloc_obj_t *cur_child = NULL;
hwloc_obj_type_t hwloc_key;
int depth;
int num_found;
#if 0
char str[128];
#endif
cur_child = (hwloc_obj_t*)malloc(sizeof(hwloc_obj_t) * 1);
/*
* Convert LAMA key to HWLOC key
*/
rmaps_lama_convert_lama_key_to_hwloc_key(lama_key, &hwloc_key, &depth);
/*
* Decend tree looking for the n'th matching subtree
*/
num_found = -1;
ret = rmaps_lama_find_nth_subtree_match_core(hwloc_topo,
parent_obj,
nth,
&num_found,
hwloc_key,
depth,
cur_child);
/*
* Check to see if we found it
*/
#if 0
hwloc_obj_snprintf(str, sizeof(str), hwloc_topo, *cur_child, "#", 0);
if( nth == num_found ) {
printf("--> FOUND : %-20s \t -- \t %2d of %2d\n", str, nth, num_found);
}
else {
printf("--> MISSING : %-20s \t -- \t %2d of %2d\n", str, nth, num_found);
}
#endif
if( nth == num_found ) {
return cur_child;
}
else {
free(cur_child);
return NULL;
}
}
static void hwloc_children(hwloc_topology_t topology, hwloc_obj_t obj,
int depth)
{
char string[128];
unsigned i;
hwloc_obj_snprintf(string, sizeof(string), topology, obj, "#", 0);
debug3("%*s%s", 2*depth, "", string);
for (i = 0; i < obj->arity; i++) {
hwloc_children(topology, obj->children[i], depth + 1);
}
}
static void find_and_assign_combinations(hwloc_obj_t obj, unsigned min, unsigned max, unsigned synthesize_arity)
{
char name[64];
unsigned i, n, nworkers;
int cpu_workers[STARPU_NMAXWORKERS];
struct _starpu_machine_config *config = _starpu_get_machine_config();
struct _starpu_machine_topology *topology = &config->topology;
hwloc_obj_snprintf(name, sizeof(name), topology->hwtopology, obj, "#", 0);
_STARPU_DEBUG("Looking at %s\n", name);
for (n = 0, i = 0; i < obj->arity; i++)
if (obj->children[i]->userdata)
/* it has a CPU worker */
n++;
if (n == 1)
{
/* If there is only one child, we go to the next level right away */
find_and_assign_combinations(obj->children[0], min, max, synthesize_arity);
return;
}
/* Add this object */
nworkers = 0;
find_workers(obj, cpu_workers, &nworkers);
if (nworkers >= min && nworkers <= max)
{
_STARPU_DEBUG("Adding it\n");
unsigned sched_ctx_id = starpu_sched_ctx_get_context();
if(sched_ctx_id == STARPU_NMAX_SCHED_CTXS)
sched_ctx_id = 0;
struct starpu_worker_collection* workers = starpu_sched_ctx_get_worker_collection(sched_ctx_id);
int newworkerid = starpu_combined_worker_assign_workerid(nworkers, cpu_workers);
STARPU_ASSERT(newworkerid >= 0);
workers->add(workers,newworkerid);
}
/* Add artificial intermediate objects recursively */
synthesize_intermediate_workers(obj->children, min, max, obj->arity, n, synthesize_arity);
/* And recurse */
for (i = 0; i < obj->arity; i++)
if (obj->children[i]->userdata == (void*) -1)
find_and_assign_combinations(obj->children[i], min, max, synthesize_arity);
}
/*
*Prints the processing units hierarchy of the machine
*Recursive function that goes throught the machine topology object
*an group them into hierarchical groups
* topology: the HWLOC object
* obj: the current object in the topology
* depth: the horizontal level in the machine topology
*/
void print_children_pu(hwloc_topology_t topology, hwloc_obj_t obj, int depth)
{
char string[128], out_string[128];
unsigned i;
if(obj->type == HWLOC_OBJ_SOCKET || obj->type == HWLOC_OBJ_CORE ||
obj->type == HWLOC_OBJ_PU){
hwloc_obj_snprintf(string, sizeof(string), topology, obj, "#", 0);
sprintf(out_string,"%*s%s\n", depth, "", string);
strcat(console_output,out_string);
}
for (i = 0; i < obj->arity; i++) {
print_children_pu(topology, obj->children[i], depth + 1);
}
}
/*
*Prints the memory hierachy of the machine
*Recursive function that goes throught the machine topology object
*an group them into hierarchical groups
* topology: the HWLOC object
* obj: the current object of the machine
*/
void print_children_mem(hwloc_topology_t topology, hwloc_obj_t obj, int depth)
{
char string[128], out_string[128];
unsigned i;
if(obj->type == HWLOC_OBJ_MACHINE || obj->type == HWLOC_OBJ_NODE ||
( obj->type == HWLOC_OBJ_CACHE && obj->arity<=1)){
hwloc_obj_snprintf(string, sizeof(string), topology, obj, "#", 0);
sprintf(out_string,"%*s%s\n", depth, "", string);
strcat(console_output,out_string);
}
else if (obj->type == HWLOC_OBJ_CACHE && obj->arity>1){
hwloc_obj_type_snprintf(string, sizeof(string), obj, 0);
sprintf(out_string,"%*s%s", depth, "", string);
strcat(console_output,out_string);
sprintf(out_string," (%dMB)\n", obj->attr->cache.size/(1024*1024));
strcat(console_output,out_string);
}
for (i = 0; i < obj->arity; i++) {
print_children_mem(topology, obj->children[i], depth + 1);
}
}
/*
*Prints one branch of the machine topology
*Recursive function that goes throught the machine topology object
*an group them into hierarchical groups
* topology: the HWLOC object that represents the machine
* obj: the current object of a level
*/
void print_machine_branch(hwloc_topology_t topology, hwloc_obj_t obj, int depth, int obj_type)
{
char string[128], out_string[128];
unsigned i,arity;
int *devIds,devId,countDev;
if (obj->type != HWLOC_OBJ_MACHINE ){
if(obj->type == HWLOC_OBJ_NODE){
hwloc_obj_snprintf(string, sizeof(string), topology, obj, "#", 0);
sprintf(out_string,"%*s%s\n", depth, "", string);
strcat(console_output,out_string);
#if defined (__DBCSR_ACC) || defined (__PW_CUDA)
if ((local_topo->ngpus > 0) && (local_topo->ngpus < local_topo->ncores)){
ma_get_nDevcu(obj->logical_index, &countDev);
devIds = malloc (countDev*sizeof(int));
ma_get_cu(obj->logical_index, devIds);
strcat(console_output," Shared GPUS: ");
for (i = 0; i<countDev; i++){
devId = devIds[i];
sprintf(out_string,"#%d ", devId);
strcat(console_output,out_string);}
strcat(console_output,"\n");
}
#endif
}
else if (obj->type == HWLOC_OBJ_SOCKET){
hwloc_obj_snprintf(string, sizeof(string), topology, obj, "#", 0);
sprintf(out_string,"%*s%s\n", depth, "", string);
strcat(console_output,out_string);}
else {
hwloc_obj_snprintf(string, sizeof(string), topology, obj, "#", 0);
if(obj->type == HWLOC_OBJ_PU) {
#if defined (__DBCSR_ACC) || defined (__PW_CUDA)
sprintf(out_string,"%*s%s\t", depth, "", string);
strcat(console_output,out_string);
if (local_topo->ngpus > 0 && local_topo->ngpus == local_topo->ncores){
ma_get_core_cu(obj->logical_index, &devId);
strcat(console_output," GPU: ");
sprintf(out_string,"%d ", devId);
strcat(console_output,out_string);}
strcat(console_output,"\n");
#else
sprintf(out_string,"%*s%s\n", depth, "", string);
strcat(console_output,out_string);
#endif
}
else if (obj->type == HWLOC_OBJ_CACHE && obj->arity>1){
hwloc_obj_type_snprintf(string, sizeof(string), obj, 0);
sprintf(out_string,"%*s%s", depth, "", string);
strcat(console_output,out_string);
sprintf(out_string," (%dMB)\n", obj->attr->cache.size/(1024*1024));
strcat(console_output,out_string);
}
else {
sprintf(out_string,"%*s%s\t", depth, "", string);
strcat(console_output,out_string);
}
}
}
if (obj->type != HWLOC_OBJ_PU) {//it is not a PU
if((obj->first_child && obj->first_child->type == HWLOC_OBJ_PU) ||
obj->first_child->type == obj_type)
arity = 1; //number of children
else
arity = obj->arity;
for (i = 0; i < arity; i++)
print_machine_branch(topology, obj->children[i],depth+1,obj_type);
}
}
/*
*Prints the machine hierarchy
*Recursive function that goes throught the machine topology object
*an group them into hierarchical groups
* topology: the HWLOC object
* obj: the current object in the topology
* depth: the horizontal level in the machine topology
*/
void print_machine(hwloc_topology_t topo, hwloc_obj_t obj, int depth)
{
char string[256], out_string[256];
unsigned i,arity;
int *devIds,devId,countDev;
if(obj->type == HWLOC_OBJ_SOCKET || obj->type == HWLOC_OBJ_MACHINE ){
hwloc_obj_snprintf(string, sizeof(string), topology, obj, "#", 0);
sprintf(out_string,"%*s%s\n", depth, "", string);
strcat(console_output,out_string);
}
else if (obj->type == HWLOC_OBJ_NODE){
hwloc_obj_snprintf(string, sizeof(string), topology, obj, "#", 0);
sprintf(out_string,"%*s%s\n", depth, "", string);
strcat(console_output,out_string);
//if the machine has shared GPUs
#if defined (__DBCSR_ACC) || defined (__PW_CUDA)
if ((local_topo->ngpus > 0) && (local_topo->ngpus < local_topo->ncores)){
ma_get_nDevcu(obj->logical_index, &countDev);
devIds = malloc (countDev*sizeof(int));
ma_get_cu(obj->logical_index, devIds);
strcat(console_output," Shared GPUS: ");
for (i = 0; i<countDev; i++){
devId = devIds[i];
sprintf(out_string,"#%d ", devId);
strcat(console_output,out_string);}
strcat(console_output,"\n");
}
#endif
}
else {
hwloc_obj_snprintf(string, sizeof(string), topology, obj, "#", 0);
if(obj->type == HWLOC_OBJ_PU )
{
#if defined (__DBCSR_ACC) || defined (__PW_CUDA)
sprintf(out_string,"%*s%s\t", depth, "", string);
strcat(console_output,out_string);
if (local_topo->ngpus > 0 && local_topo->ngpus == local_topo->ncores){
ma_get_core_cu(obj->logical_index, &devId);
strcat(console_output," GPU: ");
sprintf(out_string,"%d ", devId);
strcat(console_output,out_string);}
strcat(console_output,"\n");
#else
sprintf(out_string,"%*s%s\n", depth, "", string);
strcat(console_output,out_string);
#endif
}
else if (obj->type == HWLOC_OBJ_CACHE && obj->arity>1 ){
hwloc_obj_type_snprintf(string, sizeof(string), obj, 0);
sprintf(out_string,"%*s%s", depth, "", string);
strcat(console_output,out_string);
sprintf(out_string," (%dMB)\n", obj->attr->cache.size/(1024*1024));
strcat(console_output,out_string);
}
else if (obj->type == HWLOC_OBJ_OS_DEVICE ||
obj->type == HWLOC_OBJ_PCI_DEVICE ||
obj->type == HWLOC_OBJ_BRIDGE){
if(obj->attr->osdev.type == HWLOC_OBJ_OSDEV_NETWORK ){
sprintf(out_string,"%*s%s\n", depth, "--", "Network Card");
strcat(console_output,out_string);}
}
else if (obj->type == HWLOC_OBJ_CORE)
{
char number[33];
strcpy(string,"Core#");
sprintf(number,"%d",obj->logical_index);
strcat(string,number);
sprintf(out_string,"%*s%s\t", depth, "", string);
strcat(console_output,out_string);
}
else {
sprintf(out_string,"%*s%s\t", depth, "", string);
strcat(console_output,out_string);
}
}
if (obj->type != HWLOC_OBJ_PU) {//it is not a PU
if((obj->first_child && obj->first_child->type == HWLOC_OBJ_PU))
arity = 1; //number of children
else
arity = obj->arity;
for (i = 0; i < arity; i++)
print_machine(topo, obj->children[i],depth+1);
}
}