int main(int argc, char *argv[])
{
struct pqos_config cfg;
const struct pqos_cpuinfo *p_cpu = NULL;
const struct pqos_cap *p_cap = NULL;
unsigned sock_count, sockets[PQOS_MAX_SOCKETS];
int ret, exit_val = EXIT_SUCCESS;
memset(&cfg, 0, sizeof(cfg));
cfg.fd_log = STDOUT_FILENO;
cfg.verbose = 0;
/* PQoS Initialization - Check and initialize CAT and CMT capability */
ret = pqos_init(&cfg);
if (ret != PQOS_RETVAL_OK) {
printf("Error initializing PQoS library!\n");
exit_val = EXIT_FAILURE;
goto error_exit;
}
/* Get CMT capability and CPU info pointer */
ret = pqos_cap_get(&p_cap, &p_cpu);
if (ret != PQOS_RETVAL_OK) {
printf("Error retrieving PQoS capabilities!\n");
exit_val = EXIT_FAILURE;
goto error_exit;
}
/* Get CPU socket information to set COS */
ret = pqos_cpu_get_sockets(p_cpu, PQOS_MAX_SOCKETS,
&sock_count,
sockets);
if (ret != PQOS_RETVAL_OK) {
printf("Error retrieving CPU socket information!\n");
exit_val = EXIT_FAILURE;
goto error_exit;
}
/* Get input from user */
allocation_get_input(argc, argv);
if (sel_l3ca_cos_num != 0) {
/* Set bit mask for COS allocation */
ret = set_allocation_class(sock_count, sockets);
if (ret < 0) {
printf("Allocation configuration error!\n");
goto error_exit;
}
printf("Allocation configuration altered.\n");
}
/* Print COS and associated bit mask */
ret = print_allocation_config(sock_count, sockets);
if (ret != PQOS_RETVAL_OK) {
printf("Allocation capability not detected!\n");
exit_val = EXIT_FAILURE;
goto error_exit;
}
error_exit:
/* reset and deallocate all the resources */
ret = pqos_fini();
if (ret != PQOS_RETVAL_OK)
printf("Error shutting down PQoS library!\n");
return exit_val;
}
int main(int argc, char *argv[])
{
struct pqos_config cfg;
const struct pqos_cpuinfo *p_cpu = NULL;
const struct pqos_cap *p_cap = NULL;
const struct pqos_capability *cap_l3ca = NULL;
unsigned sock_count, *sockets = NULL;
int ret, exit_val = EXIT_SUCCESS;
memset(&cfg, 0, sizeof(cfg));
cfg.fd_log = STDOUT_FILENO;
cfg.verbose = 0;
/* PQoS Initialization - Check and initialize CAT and CMT capability */
ret = pqos_init(&cfg);
if (ret != PQOS_RETVAL_OK) {
printf("Error initializing PQoS library!\n");
exit_val = EXIT_FAILURE;
goto error_exit;
}
/* Get CMT capability and CPU info pointer */
ret = pqos_cap_get(&p_cap, &p_cpu);
if (ret != PQOS_RETVAL_OK) {
printf("Error retrieving PQoS capabilities!\n");
exit_val = EXIT_FAILURE;
goto error_exit;
}
if (argc == 2 && (!strcmp(argv[1], "-h") || !strcmp(argv[1], "-H"))) {
printf("Usage: %s\n\n", argv[0]);
goto error_exit;
}
/* Reset Api */
ret = pqos_alloc_reset(PQOS_REQUIRE_CDP_ANY, PQOS_REQUIRE_CDP_ANY,
PQOS_MBA_ANY);
if (ret != PQOS_RETVAL_OK)
printf("CAT reset failed!\n");
else
printf("CAT reset successful\n");
/* Get CPU socket information to set COS */
sockets = pqos_cpu_get_sockets(p_cpu, &sock_count);
if (sockets == NULL) {
printf("Error retrieving CPU socket information!\n");
exit_val = EXIT_FAILURE;
goto error_exit;
}
(void) pqos_cap_get_type(p_cap, PQOS_CAP_TYPE_L3CA, &cap_l3ca);
/* Print COS and associated cores */
print_allocation_config(cap_l3ca, sock_count, sockets, p_cpu);
error_exit:
/* reset and deallocate all the resources */
ret = pqos_fini();
if (ret != PQOS_RETVAL_OK)
printf("Error shutting down PQoS library!\n");
if (sockets != NULL)
free(sockets);
return exit_val;
}
/**
* @brief Prints information about cache allocation settings in the system
*/
static void
print_allocation_config(void)
{
int ret;
unsigned i;
unsigned sock_count, *sockets = NULL;
const struct pqos_cpuinfo *p_cpu = NULL;
const struct pqos_cap *p_cap = NULL;
/* Get CMT capability and CPU info pointer */
ret = pqos_cap_get(&p_cap, &p_cpu);
if (ret != PQOS_RETVAL_OK) {
printf("Error retrieving PQoS capabilities!\n");
return;
}
/* Get CPU socket information to set COS */
sockets = pqos_cpu_get_sockets(p_cpu, &sock_count);
if (sockets == NULL) {
printf("Error retrieving CPU socket information!\n");
return;
}
for (i = 0; i < sock_count; i++) {
unsigned *lcores = NULL;
unsigned lcount = 0, n = 0;
lcores = pqos_cpu_get_cores(p_cpu, sockets[i], &lcount);
if (lcores == NULL || lcount == 0) {
printf("Error retrieving core information!\n");
free(sockets);
return;
}
printf("Core information for socket %u:\n",
sockets[i]);
for (n = 0; n < lcount; n++) {
unsigned class_id = 0;
ret = pqos_alloc_assoc_get(lcores[n], &class_id);
if (ret == PQOS_RETVAL_OK)
printf(" Core %u => COS%u\n",
lcores[n], class_id);
else
printf(" Core %u => ERROR\n",
lcores[n]);
}
free(lcores);
}
free(sockets);
}
int
pqos_l3ca_reset(const struct pqos_cap *cap,
const struct pqos_cpuinfo *cpu)
{
unsigned *sockets = NULL;
unsigned sockets_num = 0, sockets_num_ret = 0;
const struct pqos_capability *item = NULL;
int ret = PQOS_RETVAL_OK;
unsigned j;
ASSERT(cap != NULL && cpu != NULL);
if (cap == NULL || cpu == NULL)
return PQOS_RETVAL_PARAM;
ret = pqos_cap_get_type(cap, PQOS_CAP_TYPE_L3CA, &item);
if (ret != PQOS_RETVAL_OK)
return ret; /**< no L3CA capability */
ASSERT(item != NULL);
/**
* Figure out number of sockets in the system
* - allocate enough memory to accommodate all socket id's
* - use stack frame allocator for it (not heap)
* - get list of socket id's through another API
* - validate that number of socket id's obtained in
* two different ways match
*/
sockets_num = __get_num_sockets(cpu);
if (sockets_num == 0)
return PQOS_RETVAL_RESOURCE;
sockets = alloca(sockets_num * sizeof(sockets[0]));
if (sockets == NULL)
return PQOS_RETVAL_RESOURCE;
ret = pqos_cpu_get_sockets(cpu, sockets_num, &sockets_num_ret, sockets);
if (ret != PQOS_RETVAL_OK)
return ret;
ASSERT(sockets_num_ret == sockets_num);
if (sockets_num != sockets_num_ret)
return PQOS_RETVAL_ERROR;
/**
* Change COS definition on all sockets
* so that each COS allows for access to all cache ways
*/
for (j = 0; j < sockets_num; j++) {
const uint64_t ways_mask = (1ULL<<item->u.l3ca->num_ways)-1ULL;
unsigned i;
for (i = 0; i < item->u.l3ca->num_classes; i++) {
struct pqos_l3ca cos;
cos.cdp = 0;
cos.class_id = i;
cos.ways_mask = ways_mask;
ret = pqos_l3ca_set(sockets[j], 1, &cos);
if (ret != PQOS_RETVAL_OK)
return ret;
}
}
/**
* Associate all cores with COS0
*/
for (j = 0; j < cpu->num_cores; j++) {
ret = pqos_l3ca_assoc_set(cpu->cores[j].lcore, 0);
if (ret != PQOS_RETVAL_OK)
return ret;
}
return ret;
}
int main(int argc, char **argv)
{
struct pqos_config cfg;
const struct pqos_cpuinfo *p_cpu = NULL;
const struct pqos_cap *p_cap = NULL;
const struct pqos_capability *cap_mon = NULL, *cap_l3ca = NULL;
unsigned sock_count, sockets[PQOS_MAX_SOCKETS];
int cmd, ret, exit_val = EXIT_SUCCESS;
int opt_index = 0;
m_cmd_name = argv[0];
print_warning();
while ((cmd = getopt_long(argc, argv,
"Hhf:i:m:Tt:l:o:u:e:c:a:p:S:srvVR",
long_cmd_opts, &opt_index)) != -1) {
switch (cmd) {
case 'h':
print_help(1);
return EXIT_SUCCESS;
case 'H':
profile_l3ca_list(stdout);
return EXIT_SUCCESS;
case 'S':
selfn_set_config(optarg);
break;
case 'f':
if (sel_config_file != NULL) {
printf("Only one config file argument is "
"accepted!\n");
return EXIT_FAILURE;
}
selfn_strdup(&sel_config_file, optarg);
parse_config_file(sel_config_file);
break;
case 'i':
selfn_monitor_interval(optarg);
break;
case 'p':
selfn_monitor_pids(optarg);
break;
case 'm':
selfn_monitor_cores(optarg);
break;
case 't':
selfn_monitor_time(optarg);
break;
case 'T':
selfn_monitor_top_like(NULL);
break;
case 'l':
selfn_log_file(optarg);
break;
case 'o':
selfn_monitor_file(optarg);
break;
case 'u':
selfn_monitor_file_type(optarg);
break;
case 'e':
selfn_allocation_class(optarg);
break;
case 'r':
sel_free_in_use_rmid = 1;
break;
case 'R':
selfn_reset_cat(NULL);
break;
case 'a':
selfn_allocation_assoc(optarg);
break;
case 'c':
selfn_allocation_select(optarg);
break;
case 's':
selfn_show_allocation(NULL);
break;
case 'v':
selfn_verbose_mode(NULL);
break;
case 'V':
selfn_super_verbose_mode(NULL);
break;
default:
printf("Unsupported option: -%c. "
"See option -h for help.\n", optopt);
return EXIT_FAILURE;
break;
case '?':
print_help(0);
return EXIT_SUCCESS;
break;
}
}
memset(&cfg, 0, sizeof(cfg));
cfg.verbose = sel_verbose_mode;
cfg.free_in_use_rmid = sel_free_in_use_rmid;
cfg.cdp_cfg = selfn_cdp_config;
/**
* Set up file descriptor for message log
*/
if (sel_log_file == NULL) {
cfg.fd_log = STDOUT_FILENO;
} else {
cfg.fd_log = open(sel_log_file, O_WRONLY|O_CREAT,
S_IRUSR|S_IWUSR);
if (cfg.fd_log == -1) {
printf("Error opening %s log file!\n", sel_log_file);
exit_val = EXIT_FAILURE;
goto error_exit_2;
}
}
ret = pqos_init(&cfg);
if (ret != PQOS_RETVAL_OK) {
printf("Error initializing PQoS library!\n");
exit_val = EXIT_FAILURE;
goto error_exit_1;
}
ret = pqos_cap_get(&p_cap, &p_cpu);
if (ret != PQOS_RETVAL_OK) {
printf("Error retrieving PQoS capabilities!\n");
exit_val = EXIT_FAILURE;
goto error_exit_2;
}
ret = pqos_cpu_get_sockets(p_cpu, PQOS_MAX_SOCKETS,
&sock_count,
sockets);
if (ret != PQOS_RETVAL_OK) {
printf("Error retrieving CPU socket information!\n");
exit_val = EXIT_FAILURE;
goto error_exit_2;
}
ret = pqos_cap_get_type(p_cap, PQOS_CAP_TYPE_MON, &cap_mon);
if (ret == PQOS_RETVAL_PARAM) {
printf("Error retrieving monitoring capabilities!\n");
exit_val = EXIT_FAILURE;
goto error_exit_2;
}
ret = pqos_cap_get_type(p_cap, PQOS_CAP_TYPE_L3CA, &cap_l3ca);
if (ret == PQOS_RETVAL_PARAM) {
printf("Error retrieving allocation capabilities!\n");
exit_val = EXIT_FAILURE;
goto error_exit_2;
}
if (sel_reset_CAT) {
/**
* Reset CAT configuration to after-reset state and exit
*/
if (pqos_l3ca_reset(p_cap, p_cpu) != PQOS_RETVAL_OK) {
exit_val = EXIT_FAILURE;
printf("CAT reset failed!\n");
} else
printf("CAT reset successful\n");
goto allocation_exit;
}
if (sel_show_allocation_config) {
/**
* Show info about allocation config and exit
*/
alloc_print_config(cap_mon, cap_l3ca, sock_count,
sockets, p_cpu);
goto allocation_exit;
}
if (sel_allocation_profile != NULL) {
if (profile_l3ca_apply(sel_allocation_profile, cap_l3ca) != 0) {
exit_val = EXIT_FAILURE;
goto error_exit_2;
}
}
switch (alloc_apply(cap_l3ca, sock_count, sockets)) {
case 0: /* nothing to apply */
break;
case 1: /* new allocation config applied and all is good */
goto allocation_exit;
break;
case -1: /* something went wrong */
default:
exit_val = EXIT_FAILURE;
goto error_exit_2;
break;
}
/**
* Just monitoring option left on the table now
*/
if (cap_mon == NULL) {
printf("Monitoring capability not detected!\n");
exit_val = EXIT_FAILURE;
goto error_exit_2;
}
if (monitor_setup(p_cpu, cap_mon) != 0) {
exit_val = EXIT_FAILURE;
goto error_exit_2;
}
monitor_loop(p_cap);
monitor_stop();
allocation_exit:
error_exit_2:
ret = pqos_fini();
ASSERT(ret == PQOS_RETVAL_OK);
if (ret != PQOS_RETVAL_OK)
printf("Error shutting down PQoS library!\n");
error_exit_1:
monitor_cleanup();
/**
* Close file descriptor for message log
*/
if (cfg.fd_log > 0 && cfg.fd_log != STDOUT_FILENO)
close(cfg.fd_log);
/**
* Free allocated memory
*/
if (sel_allocation_profile != NULL)
free(sel_allocation_profile);
if (sel_log_file != NULL)
free(sel_log_file);
if (sel_config_file != NULL)
free(sel_config_file);
return exit_val;
}
int
hw_alloc_reset(const enum pqos_cdp_config l3_cdp_cfg)
{
unsigned *sockets = NULL;
unsigned sockets_num = 0;
unsigned *l2ids = NULL;
unsigned l2id_num = 0;
const struct pqos_capability *alloc_cap = NULL;
const struct pqos_cap_l3ca *l3_cap = NULL;
const struct pqos_cap_l2ca *l2_cap = NULL;
const struct pqos_cap_mba *mba_cap = NULL;
int ret = PQOS_RETVAL_OK;
unsigned max_l3_cos = 0;
unsigned j;
if (l3_cdp_cfg != PQOS_REQUIRE_CDP_ON &&
l3_cdp_cfg != PQOS_REQUIRE_CDP_OFF &&
l3_cdp_cfg != PQOS_REQUIRE_CDP_ANY) {
LOG_ERROR("Unrecognized L3 CDP configuration setting %d!\n",
l3_cdp_cfg);
return PQOS_RETVAL_PARAM;
}
/* Get L3 CAT capabilities */
(void) pqos_cap_get_type(m_cap, PQOS_CAP_TYPE_L3CA, &alloc_cap);
if (alloc_cap != NULL)
l3_cap = alloc_cap->u.l3ca;
/* Get L2 CAT capabilities */
alloc_cap = NULL;
(void) pqos_cap_get_type(m_cap, PQOS_CAP_TYPE_L2CA, &alloc_cap);
if (alloc_cap != NULL)
l2_cap = alloc_cap->u.l2ca;
/* Get MBA capabilities */
alloc_cap = NULL;
(void) pqos_cap_get_type(m_cap, PQOS_CAP_TYPE_MBA, &alloc_cap);
if (alloc_cap != NULL)
mba_cap = alloc_cap->u.mba;
/* Check if either L2 CAT, L3 CAT or MBA is supported */
if (l2_cap == NULL && l3_cap == NULL && mba_cap == NULL) {
LOG_ERROR("L2 CAT/L3 CAT/MBA not present!\n");
ret = PQOS_RETVAL_RESOURCE; /* no L2/L3 CAT present */
goto pqos_alloc_reset_exit;
}
/* Check L3 CDP requested while not present */
if (l3_cap == NULL && l3_cdp_cfg != PQOS_REQUIRE_CDP_ANY) {
LOG_ERROR("L3 CDP setting requested but no L3 CAT present!\n");
ret = PQOS_RETVAL_RESOURCE;
goto pqos_alloc_reset_exit;
}
if (l3_cap != NULL) {
/* Check against erroneous CDP request */
if (l3_cdp_cfg == PQOS_REQUIRE_CDP_ON && !l3_cap->cdp) {
LOG_ERROR("CAT/CDP requested but not supported by the "
"platform!\n");
ret = PQOS_RETVAL_PARAM;
goto pqos_alloc_reset_exit;
}
/* Get maximum number of L3 CAT classes */
max_l3_cos = l3_cap->num_classes;
if (l3_cap->cdp && l3_cap->cdp_on)
max_l3_cos = max_l3_cos * 2;
}
/**
* Get number & list of sockets in the system
*/
sockets = pqos_cpu_get_sockets(m_cpu, &sockets_num);
if (sockets == NULL || sockets_num == 0)
goto pqos_alloc_reset_exit;
if (l3_cap != NULL) {
/**
* Change L3 COS definition on all sockets
* so that each COS allows for access to all cache ways
*/
for (j = 0; j < sockets_num; j++) {
unsigned core = 0;
ret = pqos_cpu_get_one_core(m_cpu, sockets[j], &core);
if (ret != PQOS_RETVAL_OK)
goto pqos_alloc_reset_exit;
const uint64_t ways_mask =
(1ULL << l3_cap->num_ways) - 1ULL;
ret = alloc_cos_reset(PQOS_MSR_L3CA_MASK_START,
max_l3_cos, core, ways_mask);
if (ret != PQOS_RETVAL_OK)
goto pqos_alloc_reset_exit;
}
}
if (l2_cap != NULL) {
/**
* Get number & list of L2ids in the system
* Then go through all L2 ids and reset L2 classes on them
*/
l2ids = pqos_cpu_get_l2ids(m_cpu, &l2id_num);
if (l2ids == NULL || l2id_num == 0)
goto pqos_alloc_reset_exit;
for (j = 0; j < l2id_num; j++) {
const uint64_t ways_mask =
(1ULL << l2_cap->num_ways) - 1ULL;
unsigned core = 0;
ret = pqos_cpu_get_one_by_l2id(m_cpu, l2ids[j], &core);
if (ret != PQOS_RETVAL_OK)
goto pqos_alloc_reset_exit;
ret = alloc_cos_reset(PQOS_MSR_L2CA_MASK_START,
l2_cap->num_classes, core,
ways_mask);
if (ret != PQOS_RETVAL_OK)
goto pqos_alloc_reset_exit;
}
}
if (mba_cap != NULL) {
/**
* Go through all sockets and reset MBA class defintions
* 0 is the default MBA COS value in linear mode.
*/
for (j = 0; j < sockets_num; j++) {
unsigned core = 0;
ret = pqos_cpu_get_one_core(m_cpu, sockets[j], &core);
if (ret != PQOS_RETVAL_OK)
goto pqos_alloc_reset_exit;
ret = alloc_cos_reset(PQOS_MSR_MBA_MASK_START,
mba_cap->num_classes, core, 0);
if (ret != PQOS_RETVAL_OK)
goto pqos_alloc_reset_exit;
}
}
/**
* Associate all cores with COS0
*/
ret = alloc_assoc_reset();
if (ret != PQOS_RETVAL_OK)
goto pqos_alloc_reset_exit;
/**
* Turn L3 CDP ON or OFF upon the request
*/
if (l3_cap != NULL) {
if (l3_cdp_cfg == PQOS_REQUIRE_CDP_ON && !l3_cap->cdp_on) {
/**
* Turn on CDP
*/
LOG_INFO("Turning CDP ON ...\n");
ret = cdp_enable(sockets_num, sockets, 1);
if (ret != PQOS_RETVAL_OK) {
LOG_ERROR("CDP enable error!\n");
goto pqos_alloc_reset_exit;
}
_pqos_cap_l3cdp_change(l3_cap->cdp_on, 1);
}
if (l3_cdp_cfg == PQOS_REQUIRE_CDP_OFF &&
l3_cap->cdp_on && l3_cap->cdp) {
/**
* Turn off CDP
*/
LOG_INFO("Turning CDP OFF ...\n");
ret = cdp_enable(sockets_num, sockets, 0);
if (ret != PQOS_RETVAL_OK) {
LOG_ERROR("CDP disable error!\n");
goto pqos_alloc_reset_exit;
}
_pqos_cap_l3cdp_change(l3_cap->cdp_on, 0);
}
}
pqos_alloc_reset_exit:
if (sockets != NULL)
free(sockets);
if (l2ids != NULL)
free(l2ids);
return ret;
}