int process_gpu_status(
struct pbsnode *pnode,
unsigned int &i,
std::vector<std::string> &status_info)
{
pbs_attribute temp;
int gpu_count = 0;
int rc = PBSE_NONE;
char buf[MAXLINE * 2];
std::string gpu_info = "";
memset(&temp, 0, sizeof(temp));
if ((rc = decode_arst(&temp, NULL, NULL, NULL, 0)) != PBSE_NONE)
{
log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_NODE, __func__, "cannot initialize attribute");
finish_gpu_status(i, status_info);
return(rc);
}
/* move past the initial gpu status */
i++;
for (; i < status_info.size(); i++)
{
if (!strcmp(status_info[i].c_str(), CRAY_GPU_STATUS_END))
break;
if (!strncmp(status_info[i].c_str(), "gpu_id=", strlen("gpu_id=")))
{
snprintf(buf, sizeof(buf), "gpu[%d]=%s;", gpu_count, status_info[i].c_str());
gpu_info += buf;
gpu_count++;
}
else
{
gpu_info += status_info[i].c_str();
gpu_info += ';';
}
}
set_ngpus(pnode, gpu_count);
decode_arst(&temp, NULL, NULL, gpu_info.c_str(), 0);
node_gpustatus_list(&temp, pnode, ATR_ACTION_ALTER);
free_arst(&temp);
return(rc);
} /* END process_gpu_status() */
int process_gpu_status(
struct pbsnode *pnode,
char **str_ptr)
{
char *str = *str_ptr;
pbs_attribute temp;
int gpu_count = 0;
int rc;
char buf[MAXLINE * 2];
dynamic_string *gpu_info;
memset(&temp, 0, sizeof(temp));
if ((gpu_info = get_dynamic_string(-1, NULL)) == NULL)
{
*str_ptr = finish_gpu_status(str);
return(ENOMEM);
}
if ((rc = decode_arst(&temp, NULL, NULL, NULL, 0)) != PBSE_NONE)
{
log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_NODE, __func__, "cannot initialize attribute");
*str_ptr = finish_gpu_status(str);
free_dynamic_string(gpu_info);
return(rc);
}
/* move past the initial gpu status */
str += strlen(str) + 1;
for (; str != NULL && *str != '\0'; str += strlen(str) + 1)
{
if (!strcmp(str, CRAY_GPU_STATUS_END))
break;
if (!strncmp(str, "gpu_id=", strlen("gpu_id=")))
{
snprintf(buf, sizeof(buf), "gpu[%d]=%s;", gpu_count, str);
rc = append_dynamic_string(gpu_info, buf);
gpu_count++;
}
else
{
rc = append_dynamic_string(gpu_info, str);
rc = append_char_to_dynamic_string(gpu_info, ';');
}
if (rc != PBSE_NONE)
{
free_dynamic_string(gpu_info);
*str_ptr = finish_gpu_status(str);
return(rc);
}
}
set_ngpus(pnode, gpu_count);
decode_arst(&temp, NULL, NULL, gpu_info->str, 0);
node_gpustatus_list(&temp, pnode, ATR_ACTION_ALTER);
free_arst(&temp);
free_dynamic_string(gpu_info);
*str_ptr = str;
return(PBSE_NONE);
} /* END process_gpu_status() */
int is_gpustat_get(
struct pbsnode *np, /* I (modified) */
unsigned int &i,
std::vector<std::string> &status_info)
{
pbs_attribute temp;
const char *gpuid = NULL;
char log_buf[LOCAL_LOG_BUF_SIZE];
int gpuidx = -1;
std::stringstream gpuinfo;
int need_delimiter = FALSE;
int reportedgpucnt = 0;
int startgpucnt = 0;
int drv_ver = 0;
if (np == NULL)
{
sprintf(log_buf, "Invalid parameter for np passed to is_gpustat_get");
log_event(PBSEVENT_SYSTEM, PBS_EVENTCLASS_SERVER, __func__, log_buf);
return(PBSE_BAD_PARAMETER);
}
if (LOGLEVEL >= 7)
{
sprintf(log_buf, "received gpu status from node %s", np->nd_name);
log_record(PBSEVENT_SCHED, PBS_EVENTCLASS_REQUEST, __func__, log_buf);
}
/* save current gpu count for node */
startgpucnt = np->nd_ngpus;
/*
* Before filling the "temp" pbs_attribute, initialize it.
* The second and third parameter to decode_arst are never
* used, so just leave them empty. (GBS)
*/
memset(&temp, 0, sizeof(temp));
if (decode_arst(&temp, NULL, NULL, NULL, 0))
{
DBPRT(("is_gpustat_get: cannot initialize attribute\n"));
return(DIS_NOCOMMIT);
}
i++;
for (; i < status_info.size(); i++)
{
/* add the info to the "temp" attribute */
const char *str = status_info[i].c_str();
/* get timestamp */
if (!strncmp(str, "timestamp=", 10))
{
if (decode_arst(&temp, NULL, NULL, str, 0))
{
DBPRT(("is_gpustat_get: cannot add attributes\n"));
free_arst(&temp);
move_past_gpu_status(i, status_info);
return(DIS_NOCOMMIT);
}
continue;
}
/* get driver version, if there is one */
if (!strncmp(str, "driver_ver=", 11))
{
if (decode_arst(&temp, NULL, NULL, str, 0))
{
DBPRT(("is_gpustat_get: cannot add attributes\n"));
free_arst(&temp);
move_past_gpu_status(i, status_info);
return(DIS_NOCOMMIT);
}
drv_ver = atoi(str + 11);
continue;
}
else if (!strcmp(str, END_GPU_STATUS))
{
break;
}
/* gpuid must come before the rest or we will be in trouble */
if (!strncmp(str, "gpuid=", 6))
{
if (gpuinfo.str().size() > 0)
{
if (decode_arst(&temp, NULL, NULL, gpuinfo.str().c_str(), 0))
{
DBPRT(("is_gpustat_get: cannot add attributes\n"));
free_arst(&temp);
move_past_gpu_status(i, status_info);
return(DIS_NOCOMMIT);
}
gpuinfo.str("");
}
gpuid = &str[6];
/*
* Get this gpus index, if it does not yet exist then find an empty entry.
* We need to allow for the gpu status results being returned in
* different orders since the nvidia order may change upon mom's reboot
*/
gpuidx = gpu_entry_by_id(np, gpuid, TRUE);
if (gpuidx == -1)
{
/*
* Failure - we could not get / create a nd_gpusn entry for this gpu,
* log an error message.
*/
if (LOGLEVEL >= 3)
{
sprintf(log_buf,
"Failed to get/create entry for gpu %s on node %s\n",
gpuid,
np->nd_name);
log_ext(-1, __func__, log_buf, LOG_DEBUG);
}
free_arst(&temp);
move_past_gpu_status(i, status_info);
return(DIS_SUCCESS);
}
gpuinfo << "gpu[" << gpuidx << "]=gpu_id=" << gpuid << ";";
need_delimiter = FALSE;
reportedgpucnt++;
np->nd_gpusn[gpuidx].driver_ver = drv_ver;
/* mark that this gpu node is not virtual */
np->nd_gpus_real = TRUE;
/*
* if we have not filled in the gpu_id returned by the mom node
* then fill it in
*/
if ((gpuidx >= 0) && (np->nd_gpusn[gpuidx].gpuid == NULL))
{
np->nd_gpusn[gpuidx].gpuid = strdup(gpuid);
}
}
else
{
if (need_delimiter)
{
gpuinfo << ";";
}
gpuinfo << str;
need_delimiter = TRUE;
}
/* check current gpu mode and determine gpu state */
if (!memcmp(str, "gpu_mode=", 9))
{
if ((!memcmp(str + 9, "Normal", 6)) || (!memcmp(str + 9, "Default", 7)))
{
np->nd_gpusn[gpuidx].mode = gpu_normal;
if (gpu_has_job(np, gpuidx))
{
np->nd_gpusn[gpuidx].state = gpu_shared;
}
else
{
np->nd_gpusn[gpuidx].inuse = 0;
np->nd_gpusn[gpuidx].state = gpu_unallocated;
}
}
else if ((!memcmp(str + 9, "Exclusive", 9)) ||
(!memcmp(str + 9, "Exclusive_Thread", 16)))
{
np->nd_gpusn[gpuidx].mode = gpu_exclusive_thread;
if (gpu_has_job(np, gpuidx))
{
np->nd_gpusn[gpuidx].state = gpu_exclusive;
}
else
{
np->nd_gpusn[gpuidx].inuse = 0;
np->nd_gpusn[gpuidx].state = gpu_unallocated;
}
}
else if (!memcmp(str + 9, "Exclusive_Process", 17))
{
np->nd_gpusn[gpuidx].mode = gpu_exclusive_process;
if (gpu_has_job(np, gpuidx))
{
np->nd_gpusn[gpuidx].state = gpu_exclusive;
}
else
{
np->nd_gpusn[gpuidx].inuse = 0;
np->nd_gpusn[gpuidx].state = gpu_unallocated;
}
}
else if (!memcmp(str + 9, "Prohibited", 10))
{
np->nd_gpusn[gpuidx].mode = gpu_prohibited;
np->nd_gpusn[gpuidx].state = gpu_unavailable;
}
else
{
/* unknown mode, default to prohibited */
np->nd_gpusn[gpuidx].mode = gpu_prohibited;
np->nd_gpusn[gpuidx].state = gpu_unavailable;
if (LOGLEVEL >= 3)
{
sprintf(log_buf,
"GPU %s has unknown mode on node %s",
gpuid,
np->nd_name);
log_ext(-1, __func__, log_buf, LOG_DEBUG);
}
}
/* add gpu_mode so it gets added to the pbs_attribute */
if (need_delimiter)
{
gpuinfo << ";";
}
switch (np->nd_gpusn[gpuidx].state)
{
case gpu_unallocated:
gpuinfo << "gpu_state=Unallocated";
break;
case gpu_shared:
gpuinfo << "gpu_state=Shared";
break;
case gpu_exclusive:
gpuinfo << "gpu_state=Exclusive";
break;
case gpu_unavailable:
gpuinfo << "gpu_state=Unavailable";
break;
}
}
} /* end of while disrst */
if (gpuinfo.str().size() > 0)
{
if (decode_arst(&temp, NULL, NULL, gpuinfo.str().c_str(), 0))
{
DBPRT(("is_gpustat_get: cannot add attributes\n"));
free_arst(&temp);
move_past_gpu_status(i, status_info);
return(DIS_NOCOMMIT);
}
}
/* maintain the gpu count, if it has changed we need to update the nodes file */
if (reportedgpucnt != startgpucnt)
{
np->nd_ngpus = reportedgpucnt;
/* update the nodes file */
update_nodes_file(np);
}
node_gpustatus_list(&temp, np, ATR_ACTION_ALTER);
move_past_gpu_status(i, status_info);
return(DIS_SUCCESS);
} /* END is_gpustat_get() */