static int find_all_sockets_cores (resrc_api_ctx_t *rsapi, resrc_t *node,
int *nsocks, int *ncs)
{
json_t *reqobj= NULL;
resrc_reqst_t *req = NULL;
resrc_tree_t *st = NULL;
resrc_tree_t *ct = NULL;
reqobj = Jnew ();
create_req4allsocks (reqobj);
req = resrc_reqst_from_json (rsapi, reqobj, NULL);
*nsocks = resrc_tree_search (rsapi, node, req, &st, false);
resrc_reqst_destroy (rsapi, req);
resrc_tree_destroy (rsapi, st, false, false);
Jput (reqobj);
reqobj = Jnew ();
create_req4allcores (reqobj);
req = resrc_reqst_from_json (rsapi, reqobj, NULL);
*ncs = resrc_tree_search (rsapi, node, req, &ct, false);
resrc_reqst_destroy (rsapi, req);
resrc_tree_destroy (rsapi, ct, false, false);
Jput (reqobj);
return (*nsocks > 0 && *ncs > 0) ? 0 : -1;
}
/*
* find_resources() identifies the all of the resource candidates for
* the job. The set of resources returned could be more than the job
* requires. A later call to select_resources() will cull this list
* down to the most appropriate set for the job.
*
* Inputs: resrcs - hash table of all resources
* resrc_reqst - the resources the job requests
* Returns: nfound - the number of resources found
* found_tree - a resource tree containing resources that satisfy the
* job's request or NULL if none are found
*/
int64_t find_resources (flux_t *h, resrc_t *resrc, resrc_reqst_t *resrc_reqst,
resrc_tree_t **found_tree)
{
int64_t nfound = 0;
if (!resrc || !resrc_reqst) {
flux_log (h, LOG_ERR, "%s: invalid arguments", __FUNCTION__);
goto ret;
}
/*
* A start time of zero is used to restrict the search to now
* (appropriate for FCFS) and prevent any search into the future.
*/
if (resrc_reqst_set_starttime (resrc_reqst, 0) ||
resrc_reqst_set_endtime (resrc_reqst, 0))
goto ret;
*found_tree = NULL;
nfound = resrc_tree_search (resrc, resrc_reqst, found_tree, true);
if (!nfound && *found_tree) {
resrc_tree_destroy (*found_tree, false);
*found_tree = NULL;
}
ret:
return nfound;
}
/* returns the number of resource or resource composites found */
int64_t resrc_tree_search (resrc_t *resrc_in, resrc_reqst_t *resrc_reqst,
resrc_tree_t **found_tree, bool available)
{
int64_t nfound = 0;
resrc_tree_list_t *children = NULL;
resrc_tree_t *child_tree;
resrc_tree_t *new_tree = NULL;
if (!resrc_in || !found_tree || !resrc_reqst) {
goto ret;
}
if (resrc_match_resource (resrc_in, resrc_reqst, available)) {
if (resrc_reqst_num_children (resrc_reqst)) {
if (resrc_tree_num_children (resrc_phys_tree (resrc_in))) {
new_tree = resrc_tree_new (*found_tree, resrc_in);
children = resrc_tree_children (resrc_phys_tree (resrc_in));
if (match_children (children, resrc_reqst->children, new_tree,
available)) {
nfound = 1;
resrc_reqst->nfound++;
} else {
resrc_tree_destroy (new_tree, false);
}
}
} else {
(void) resrc_tree_new (*found_tree, resrc_in);
nfound = 1;
resrc_reqst->nfound++;
}
} else if (resrc_tree_num_children (resrc_phys_tree (resrc_in))) {
/*
* This clause visits the children of the current resource
* searching for a match to the resource request. The found
* tree must be extended to include this intermediate
* resource.
*
* This also allows the resource request to be sparsely
* defined. E.g., it might only stipulate a node with 4 cores
* and omit the intervening socket.
*/
if (*found_tree)
new_tree = resrc_tree_new (*found_tree, resrc_in);
else {
new_tree = resrc_tree_new (NULL, resrc_in);
*found_tree = new_tree;
}
children = resrc_tree_children (resrc_phys_tree (resrc_in));
child_tree = resrc_tree_list_first (children);
while (child_tree) {
nfound += resrc_tree_search (resrc_tree_resrc (child_tree),
resrc_reqst, &new_tree, available);
child_tree = resrc_tree_list_next (children);
}
}
ret:
return nfound;
}
int rsreader_force_link2rank (resrc_api_ctx_t *rsapi, machs_t *machs)
{
int rc = -1;
resrc_tree_t *rt = NULL;
resrc_t *r_resrc = resrc_tree_resrc (resrc_tree_root (rsapi));
if (!find_all_nodes (rsapi, r_resrc, &rt))
rc = rsreader_set_node_digest (machs, rt);
if (rt)
resrc_tree_destroy (rsapi, rt, false, false);
return rc;
}
/*
* find_resources() identifies the all of the resource candidates for
* the job. The set of resources returned could be more than the job
* requires. A later call to select_resources() will cull this list
* down to the most appropriate set for the job.
*
* Inputs: resrcs - hash table of all resources
* resrc_reqst - the resources the job requests
* Returns: nfound - the number of resources found
* found_tree - a resource tree containing resources that satisfy the
* job's request or NULL if none are found
*/
int64_t find_resources (flux_t h, resrc_t *resrc, resrc_reqst_t *resrc_reqst,
resrc_tree_t **found_tree)
{
int64_t nfound = 0;
if (!resrc || !resrc_reqst) {
flux_log (h, LOG_ERR, "%s: invalid arguments", __FUNCTION__);
goto ret;
}
*found_tree = NULL;
nfound = resrc_tree_search (resrc, resrc_reqst, found_tree, true);
if (!nfound && *found_tree) {
resrc_tree_destroy (*found_tree, false);
*found_tree = NULL;
}
ret:
return nfound;
}
static int test_using_reader_hwloc ()
{
int rc = 1;
hwloc_topology_t topology;
resrc_api_ctx_t *rsapi = NULL;
resrc_tree_t *phys_tree = NULL;
init_time();
if (!(rsapi = resrc_api_init ()))
goto ret;
ok ((hwloc_topology_init (&topology) == 0),
"hwloc topology init succeeded");
ok ((hwloc_topology_load (topology) == 0),
"hwloc topology load succeeded");
/* Generate the hardware hierarchy of resources using hwloc reader */
ok ((resrc_generate_hwloc_resources (rsapi, topology, NULL, NULL) != 0),
"resource generation from hwloc took: %lf",
((double)get_time())/1000000);
hwloc_topology_destroy (topology);
phys_tree = resrc_tree_root (rsapi);
if (!phys_tree)
goto ret;
rc = test_a_resrc (rsapi, resrc_tree_resrc (phys_tree), false);
ret:
if (rsapi) {
if (phys_tree)
resrc_tree_destroy (rsapi, phys_tree, true, true);
resrc_api_fini (rsapi);
}
return rc;
}
/*
* These is a test of the resrc library. The test uses two methods
* for generating a resrc object. The first loads an RDL-formatted
* resource identified by the TESTRESRC_INPUT_FILE environment
* variable. The second option generates a resrc object from the node
* it is running on using the hwloc library. The test then conducts a
* number of resrc library operations on each of these resrc objects.
*/
int main (int argc, char *argv[])
{
char *filename = NULL;
hwloc_topology_t topology;
int rc1 = 1, rc2 = 1;
resrc_t *resrc = NULL;
resrc_flow_t *power_flow = NULL;
resrc_flow_t *bw_flow = NULL;
plan (26 + num_temporal_allocation_tests);
test_temporal_allocation ();
if ((filename = getenv ("TESTRESRC_INPUT_FILE"))) {
ok (!(filename == NULL || *filename == '\0'), "resource file provided");
ok ((access (filename, F_OK) == 0), "resource file exists");
ok ((access (filename, R_OK) == 0), "resource file readable");
init_time();
resrc_init ();
resrc = resrc_generate_rdl_resources (filename, "default");
ok ((resrc != NULL), "resource generation from config file took: %lf",
((double)get_time())/1000000);
if (resrc) {
power_flow = resrc_flow_generate_rdl (filename, "power");
if (power_flow) {
if (verbose) {
printf ("Listing power tree\n");
resrc_flow_print (power_flow);
printf ("End of power tree\n");
}
bw_flow = resrc_flow_generate_rdl (filename, "bandwidth");
if (bw_flow) {
if (verbose) {
printf ("Listing bandwidth tree\n");
resrc_flow_print (bw_flow);
printf ("End of bandwidth tree\n");
}
} else
goto ret;
} else
goto ret;
rc1 = test_a_resrc (resrc, true);
resrc_flow_destroy (bw_flow);
resrc_flow_destroy (power_flow);
resrc_tree_destroy (resrc_phys_tree (resrc), true);
resrc_fini ();
}
}
init_time();
resrc_init ();
ok ((hwloc_topology_init (&topology) == 0),
"hwloc topology init succeeded");
ok ((hwloc_topology_load (topology) == 0),
"hwloc topology load succeeded");
ok (((resrc = resrc_create_cluster ("cluster")) != 0),
"cluster resource creation succeeded");
ok ((resrc_generate_hwloc_resources (resrc, topology, NULL, NULL) != 0),
"resource generation from hwloc took: %lf",
((double)get_time())/1000000);
hwloc_topology_destroy (topology);
if (resrc) {
rc2 = test_a_resrc (resrc, false);
resrc_tree_destroy (resrc_phys_tree (resrc), true);
resrc_fini ();
}
ret:
done_testing ();
return (rc1 | rc2);
}
static int test_a_resrc (resrc_t *resrc, bool rdl)
{
int found = 0;
int rc = 0;
int64_t nowtime = epochtime ();
JSON o = NULL;
JSON req_res = NULL;
resrc_reqst_t *resrc_reqst = NULL;
resrc_tree_t *deserialized_tree = NULL;
resrc_tree_t *found_tree = NULL;
resrc_tree_t *resrc_tree = NULL;
resrc_tree_t *selected_tree = NULL;
resrc_tree = resrc_phys_tree (resrc);
ok ((resrc_tree != NULL), "resource tree valid");
if (!resrc_tree)
goto ret;
if (verbose) {
printf ("Listing resource tree\n");
resrc_tree_print (resrc_tree);
printf ("End of resource tree\n");
}
/*
* Build a resource composite to search for. Two variants are
* constructed depending on whether the loaded resources came
* from the sample RDL file or from the hwloc. The hwloc request
* does not span multiple nodes or contain the localid property.
*/
req_res = Jnew ();
if (rdl) {
JSON bandwidth = Jnew ();
JSON child_core = Jnew ();
JSON child_sock = Jnew ();
JSON graph_array = Jnew_ar ();
JSON ja = Jnew_ar ();
JSON jpropo = Jnew (); /* json property object */
JSON memory = Jnew ();
JSON power = Jnew ();
/* JSON jtago = Jnew (); /\* json tag object *\/ */
/* Jadd_bool (jtago, "maytag", true); */
/* Jadd_bool (jtago, "yourtag", true); */
Jadd_str (memory, "type", "memory");
Jadd_int (memory, "req_qty", 1);
Jadd_int (memory, "size", 100);
json_object_array_add (ja, memory);
Jadd_str (child_core, "type", "core");
Jadd_int (child_core, "req_qty", 6);
Jadd_bool (child_core, "exclusive", true);
Jadd_int (jpropo, "localid", 1);
json_object_object_add (child_core, "properties", jpropo);
json_object_array_add (ja, child_core);
Jadd_str (child_sock, "type", "socket");
Jadd_int (child_sock, "req_qty", 2);
json_object_object_add (child_sock, "req_children", ja);
Jadd_str (bandwidth, "type", "bandwidth");
Jadd_int (bandwidth, "size", 100);
json_object_array_add (graph_array, bandwidth);
Jadd_str (power, "type", "power");
Jadd_int (power, "size", 10);
json_object_array_add (graph_array, power);
Jadd_str (req_res, "type", "node");
Jadd_int (req_res, "req_qty", 2);
Jadd_int64 (req_res, "starttime", nowtime);
/* json_object_object_add (req_res, "tags", jtago); */
json_object_object_add (req_res, "req_child", child_sock);
json_object_object_add (req_res, "graphs", graph_array);
} else {
Jadd_str (req_res, "type", "core");
Jadd_int (req_res, "req_qty", 2);
Jadd_bool (req_res, "exclusive", true);
}
resrc_reqst = resrc_reqst_from_json (req_res, NULL);
Jput (req_res);
ok ((resrc_reqst != NULL), "resource request valid");
if (!resrc_reqst)
goto ret;
if (verbose) {
printf ("Listing resource request tree\n");
resrc_reqst_print (resrc_reqst);
printf ("End of resource request tree\n");
}
init_time ();
found = resrc_tree_search (resrc, resrc_reqst, &found_tree, true);
ok (found, "found %d requested resources in %lf", found,
((double)get_time ())/1000000);
if (!found)
goto ret;
if (verbose) {
printf ("Listing found tree\n");
resrc_tree_print (found_tree);
printf ("End of found tree\n");
}
o = Jnew ();
init_time ();
rc = resrc_tree_serialize (o, found_tree);
ok (!rc, "found resource serialization took: %lf",
((double)get_time ())/1000000);
if (verbose) {
printf ("The found resources serialized: %s\n", Jtostr (o));
}
deserialized_tree = resrc_tree_deserialize (o, NULL);
if (verbose) {
printf ("Listing deserialized tree\n");
resrc_tree_print (deserialized_tree);
printf ("End of deserialized tree\n");
}
Jput (o);
init_time ();
/*
* Exercise time-based allocations for the rdl case and
* now-based allocations for the hwloc case
*/
selected_tree = test_select_resources (found_tree, NULL, 1);
if (rdl)
rc = resrc_tree_allocate (selected_tree, 1, nowtime, nowtime + 3600);
else
rc = resrc_tree_allocate (selected_tree, 1, 0, 0);
ok (!rc, "successfully allocated resources for job 1");
resrc_tree_destroy (selected_tree, false);
resrc_tree_unstage_resources (found_tree);
selected_tree = test_select_resources (found_tree, NULL, 2);
if (rdl)
rc = resrc_tree_allocate (selected_tree, 2, nowtime, nowtime + 3600);
else
rc = resrc_tree_allocate (selected_tree, 2, 0, 0);
ok (!rc, "successfully allocated resources for job 2");
resrc_tree_destroy (selected_tree, false);
resrc_tree_unstage_resources (found_tree);
selected_tree = test_select_resources (found_tree, NULL, 3);
if (rdl)
rc = resrc_tree_allocate (selected_tree, 3, nowtime, nowtime + 3600);
else
rc = resrc_tree_allocate (selected_tree, 3, 0, 0);
ok (!rc, "successfully allocated resources for job 3");
resrc_tree_destroy (selected_tree, false);
resrc_tree_unstage_resources (found_tree);
selected_tree = test_select_resources (found_tree, NULL, 4);
if (rdl)
rc = resrc_tree_reserve (selected_tree, 4, nowtime, nowtime + 3600);
else
rc = resrc_tree_reserve (selected_tree, 4, 0, 0);
ok (!rc, "successfully reserved resources for job 4");
resrc_tree_destroy (selected_tree, false);
resrc_tree_unstage_resources (found_tree);
printf (" allocate and reserve took: %lf\n",
((double)get_time ())/1000000);
if (verbose) {
printf ("Allocated and reserved resources\n");
resrc_tree_print (resrc_tree);
}
init_time ();
rc = resrc_tree_release (found_tree, 1);
ok (!rc, "resource release of job 1 took: %lf",
((double)get_time ())/1000000);
if (verbose) {
printf ("Same resources without job 1\n");
resrc_tree_print (resrc_tree);
}
init_time ();
resrc_reqst_destroy (resrc_reqst);
resrc_tree_destroy (deserialized_tree, true);
resrc_tree_destroy (found_tree, false);
printf (" destroy took: %lf\n", ((double)get_time ())/1000000);
ret:
return rc;
}
/*
* select_resources() selects from the set of resource candidates the
* best resources for the job.
*
* Inputs: found_tree - tree of resource tree candidates
* resrc_reqst - the resources the job requests
* selected_parent - parent of the selected resource tree
* Returns: a resource tree of however many resources were selected
*/
resrc_tree_t *select_resources (flux_t *h, resrc_tree_t *found_tree,
resrc_reqst_t *resrc_reqst,
resrc_tree_t *selected_parent)
{
resrc_t *resrc;
resrc_tree_list_t *children = NULL;
resrc_tree_t *child_tree;
resrc_tree_t *selected_tree = NULL;
if (!resrc_reqst) {
flux_log (h, LOG_ERR, "%s: called with empty request", __FUNCTION__);
return NULL;
}
/*
* A start time of zero is used to restrict the search to now
* (appropriate for FCFS) and prevent any search into the future.
*/
if (resrc_reqst_set_starttime (resrc_reqst, 0) ||
resrc_reqst_set_endtime (resrc_reqst, 0))
return NULL;
resrc = resrc_tree_resrc (found_tree);
if (resrc_match_resource (resrc, resrc_reqst, true)) {
if (resrc_reqst_num_children (resrc_reqst)) {
if (resrc_tree_num_children (found_tree)) {
selected_tree = resrc_tree_new (selected_parent, resrc);
if (select_children (h, resrc_tree_children (found_tree),
resrc_reqst_children (resrc_reqst),
selected_tree)) {
resrc_stage_resrc (resrc,
resrc_reqst_reqrd_size (resrc_reqst),
resrc_reqst_graph_reqs (resrc_reqst));
resrc_reqst_add_found (resrc_reqst, 1);
flux_log (h, LOG_DEBUG, "selected %s", resrc_name (resrc));
} else {
resrc_tree_destroy (selected_tree, false);
}
}
} else {
selected_tree = resrc_tree_new (selected_parent, resrc);
resrc_stage_resrc (resrc, resrc_reqst_reqrd_size (resrc_reqst),
resrc_reqst_graph_reqs (resrc_reqst));
resrc_reqst_add_found (resrc_reqst, 1);
flux_log (h, LOG_DEBUG, "selected %s", resrc_name (resrc));
}
} else if (resrc_tree_num_children (found_tree)) {
/*
* This clause visits the children of the current resource
* searching for a match to the resource request. The selected
* tree must be extended to include this intermediate
* resource.
*
* This also allows the resource request to be sparsely
* defined. E.g., it might only stipulate a node with 4 cores
* and omit the intervening socket.
*/
selected_tree = resrc_tree_new (selected_parent, resrc);
children = resrc_tree_children (found_tree);
child_tree = resrc_tree_list_first (children);
while (child_tree) {
if (select_resources (h, child_tree, resrc_reqst, selected_tree) &&
resrc_reqst_nfound (resrc_reqst) >=
resrc_reqst_reqrd_qty (resrc_reqst))
break;
child_tree = resrc_tree_list_next (children);
}
}
return selected_tree;
}
/*
* reserve_resources() reserves resources for the specified job id.
* Unlike the FCFS version where selected_tree provides the tree of
* resources to reserve, this backfill version will search into the
* future to find a time window when all of the required resources are
* available, reserve those, and return the pointer to the selected
* tree.
*/
int reserve_resources (flux_t h, resrc_tree_t **selected_tree, int64_t job_id,
int64_t starttime, int64_t walltime, resrc_t *resrc,
resrc_reqst_t *resrc_reqst)
{
int rc = -1;
int64_t *completion_time = NULL;
int64_t nfound = 0;
int64_t prev_completion_time = -1;
resrc_tree_t *found_tree = NULL;
if (reservation_depth > 0 && (curr_reservation_depth >= reservation_depth)) {
goto ret;
} else if (!resrc || !resrc_reqst) {
flux_log (h, LOG_ERR, "%s: invalid arguments", __FUNCTION__);
goto ret;
}
if (*selected_tree) {
resrc_tree_destroy (*selected_tree, false);
*selected_tree = NULL;
}
zlist_sort (completion_times, compare_int64_ascending);
for (completion_time = zlist_first (completion_times);
completion_time;
completion_time = zlist_next (completion_times)) {
/* Purge past times from consideration */
if (*completion_time < starttime) {
zlist_remove (completion_times, completion_time);
continue;
}
/* Don't test the same time multiple times */
if (prev_completion_time == *completion_time)
continue;
resrc_reqst_set_starttime (resrc_reqst, *completion_time + 1);
resrc_reqst_set_endtime (resrc_reqst, *completion_time + 1 + walltime);
flux_log (h, LOG_DEBUG, "Attempting to reserve %"PRId64" nodes for job "
"%"PRId64" at time %"PRId64"",
resrc_reqst_reqrd_qty (resrc_reqst), job_id,
*completion_time + 1);
nfound = resrc_tree_search (resrc, resrc_reqst, &found_tree, true);
if (nfound >= resrc_reqst_reqrd_qty (resrc_reqst)) {
*selected_tree = select_resources (h, found_tree, resrc_reqst, NULL);
resrc_tree_destroy (found_tree, false);
if (*selected_tree) {
rc = resrc_tree_reserve (*selected_tree, job_id,
*completion_time + 1,
*completion_time + 1 + walltime);
if (rc) {
resrc_tree_destroy (*selected_tree, false);
*selected_tree = NULL;
} else {
curr_reservation_depth++;
flux_log (h, LOG_DEBUG, "Reserved %"PRId64" nodes for job "
"%"PRId64" from %"PRId64" to %"PRId64"",
resrc_reqst_reqrd_qty (resrc_reqst), job_id,
*completion_time + 1,
*completion_time + 1 + walltime);
}
break;
}
}
prev_completion_time = *completion_time;
}
ret:
return rc;
}
static int test_using_reader_rdl ()
{
int rc = 1;
char *filename = NULL;
resrc_api_ctx_t *rsapi = NULL;
resrc_t *resrc = NULL;
resrc_tree_t *phys_tree = NULL;
resrc_flow_t *power_flow = NULL;
resrc_flow_t *bw_flow = NULL;
if (!(filename = getenv ("TESTRESRC_INPUT_FILE")))
goto ret;
ok (!(filename == NULL || *filename == '\0'), "resource file provided");
ok ((access (filename, F_OK) == 0), "resource file exists");
ok ((access (filename, R_OK) == 0), "resource file readable");
init_time();
if (!(rsapi = resrc_api_init ()))
goto ret;
/* Generate a hardware hierarchy of resources using RDL reader */
resrc = resrc_generate_rdl_resources (rsapi, filename, "default");
double e = (double)get_time()/1000000;
ok ((resrc != NULL), "resource generation from config file took: %lf", e);
if (!resrc)
goto ret;
phys_tree = resrc_tree_root (rsapi);
/* Generate a power hierarchy of resources using RDL reader */
if (!(power_flow = resrc_flow_generate_rdl (rsapi, filename, "power")))
goto ret;
else if (verbose) {
printf ("Listing power tree\n");
resrc_flow_print (power_flow);
printf ("End of power tree\n");
}
/* Generate a bandwidth hierarchy of resources using RDL reader */
if (!(bw_flow = resrc_flow_generate_rdl (rsapi, filename, "bandwidth")))
goto ret;
else if (verbose) {
printf ("Listing bandwidth tree\n");
resrc_flow_print (bw_flow);
printf ("End of bandwidth tree\n");
}
rc = test_a_resrc (rsapi, resrc, true);
ret:
if (rsapi) {
if (bw_flow)
resrc_flow_destroy (rsapi, bw_flow, true);
if (power_flow)
resrc_flow_destroy (rsapi, power_flow, true);
if (phys_tree)
resrc_tree_destroy (rsapi, phys_tree, true, true);
resrc_api_fini (rsapi);
}
return rc;
}
