/*----------------------------------------------------------------------------*/
static int
GetNumQueues()
{
FILE *fp;
char buf[MAX_PROCLINE_LEN];
int queue_cnt;
fp = fopen("/proc/interrupts", "r");
if (!fp) {
TRACE_CONFIG("Failed to read data from /proc/interrupts!\n");
return -1;
}
/* count number of NIC queues from /proc/interrupts */
queue_cnt = 0;
while (!feof(fp)) {
if (fgets(buf, MAX_PROCLINE_LEN, fp) == NULL)
break;
/* "xge0-rx" is the keyword for counting queues */
if (strstr(buf, "xge0-rx")) {
queue_cnt++;
}
}
fclose(fp);
return queue_cnt;
}
/*----------------------------------------------------------------------------*/
static void
printHelp(const char *prog_name)
{
TRACE_CONFIG("%s -p <path_to_www/> -f <mtcp_conf_file> "
"[-N num_cores] [-c <per-process core_id>] [-h]\n",
prog_name);
exit(EXIT_SUCCESS);
}
/*----------------------------------------------------------------------------*/
int
SetInterfaceInfo(char* dev_name_list)
{
struct ifreq ifr;
int eidx = 0;
int i, j;
int set_all_inf = (strncmp(dev_name_list, ALL_STRING, sizeof(ALL_STRING))==0);
TRACE_CONFIG("Loading interface setting\n");
CONFIG.eths = (struct eth_table *)
calloc(MAX_DEVICES, sizeof(struct eth_table));
if (!CONFIG.eths)
exit(EXIT_FAILURE);
if (current_iomodule_func == &ps_module_func) {
/* calculate num_devices now! */
num_devices = ps_list_devices(devices);
if (num_devices == -1) {
perror("ps_list_devices");
exit(EXIT_FAILURE);
}
/* Create socket */
int sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
if (sock == -1) {
perror("socket");
}
/* To Do: Parse dev_name_list rather than use strstr */
for (i = 0; i < num_devices; i++) {
strcpy(ifr.ifr_name, devices[i].name);
/* getting interface information */
if (ioctl(sock, SIOCGIFFLAGS, &ifr) == 0) {
if (!set_all_inf && strstr(dev_name_list, ifr.ifr_name) == NULL)
continue;
/* Setting informations */
eidx = CONFIG.eths_num++;
strcpy(CONFIG.eths[eidx].dev_name, ifr.ifr_name);
CONFIG.eths[eidx].ifindex = devices[i].ifindex;
/* getting address */
if (ioctl(sock, SIOCGIFADDR, &ifr) == 0 ) {
struct in_addr sin = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr;
CONFIG.eths[eidx].ip_addr = *(uint32_t *)&sin;
}
if (ioctl(sock, SIOCGIFHWADDR, &ifr) == 0 ) {
for (j = 0; j < ETH_ALEN; j ++) {
CONFIG.eths[eidx].haddr[j] = ifr.ifr_addr.sa_data[j];
}
}
/* Net MASK */
if (ioctl(sock, SIOCGIFNETMASK, &ifr) == 0) {
struct in_addr sin = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr;
CONFIG.eths[eidx].netmask = *(uint32_t *)&sin;
}
/* add to attached devices */
for (j = 0; j < num_devices_attached; j++) {
if (devices_attached[j] == devices[i].ifindex) {
break;
}
}
devices_attached[num_devices_attached] = devices[i].ifindex;
num_devices_attached++;
} else {
perror("SIOCGIFFLAGS");
}
}
num_queues = GetNumQueues();
if (num_queues <= 0) {
TRACE_CONFIG("Failed to find NIC queues!\n");
return -1;
}
if (num_queues > num_cpus) {
TRACE_CONFIG("Too many NIC queues available.\n");
return -1;
}
} else if (current_iomodule_func == &dpdk_module_func) {
#ifndef DISABLE_DPDK
int cpu = CONFIG.num_cores;
uint32_t cpumask = 0;
char cpumaskbuf[10];
char mem_channels[5];
int ret;
static struct ether_addr ports_eth_addr[RTE_MAX_ETHPORTS];
/* get the cpu mask */
for (ret = 0; ret < cpu; ret++)
cpumask = (cpumask | (1 << ret));
sprintf(cpumaskbuf, "%X", cpumask);
/* get the mem channels per socket */
if (CONFIG.num_mem_ch == 0) {
TRACE_ERROR("DPDK module requires # of memory channels "
"per socket parameter!\n");
exit(EXIT_FAILURE);
}
sprintf(mem_channels, "%d", CONFIG.num_mem_ch);
/* initialize the rte env first, what a waste of implementation effort! */
char *argv[] = {"",
"-c",
cpumaskbuf,
"-n",
mem_channels,
"--proc-type=auto",
""
};
const int argc = 6;
/*
* re-set getopt extern variable optind.
* this issue was a bitch to debug
* rte_eal_init() internally uses getopt() syscall
* mtcp applications that also use an `external' getopt
* will cause a violent crash if optind is not reset to zero
* prior to calling the func below...
* see man getopt(3) for more details
*/
optind = 0;
/* initialize the dpdk eal env */
ret = rte_eal_init(argc, argv);
if (ret < 0)
rte_exit(EXIT_FAILURE, "Invalid EAL args!\n");
/* give me the count of 'detected' ethernet ports */
num_devices = rte_eth_dev_count();
if (num_devices == 0) {
rte_exit(EXIT_FAILURE, "No Ethernet port!\n");
}
/* get mac addr entries of 'detected' dpdk ports */
for (ret = 0; ret < num_devices; ret++)
rte_eth_macaddr_get(ret, &ports_eth_addr[ret]);
num_queues = MIN(CONFIG.num_cores, MAX_CPUS);
struct ifaddrs *ifap;
struct ifaddrs *iter_if;
char *seek;
if (getifaddrs(&ifap) != 0) {
perror("getifaddrs: ");
exit(EXIT_FAILURE);
}
iter_if = ifap;
do {
if (iter_if->ifa_addr->sa_family == AF_INET &&
!set_all_inf &&
(seek=strstr(dev_name_list, iter_if->ifa_name)) != NULL &&
/* check if the interface was not aliased */
*(seek + strlen(iter_if->ifa_name)) != ':') {
struct ifreq ifr;
/* Setting informations */
eidx = CONFIG.eths_num++;
strcpy(CONFIG.eths[eidx].dev_name, iter_if->ifa_name);
strcpy(ifr.ifr_name, iter_if->ifa_name);
/* Create socket */
int sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
if (sock == -1) {
perror("socket");
}
/* getting address */
if (ioctl(sock, SIOCGIFADDR, &ifr) == 0 ) {
struct in_addr sin = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr;
CONFIG.eths[eidx].ip_addr = *(uint32_t *)&sin;
}
if (ioctl(sock, SIOCGIFHWADDR, &ifr) == 0 ) {
for (j = 0; j < ETH_ALEN; j ++) {
CONFIG.eths[eidx].haddr[j] = ifr.ifr_addr.sa_data[j];
}
}
/* Net MASK */
if (ioctl(sock, SIOCGIFNETMASK, &ifr) == 0) {
struct in_addr sin = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr;
CONFIG.eths[eidx].netmask = *(uint32_t *)&sin;
}
close(sock);
for (j = 0; j < num_devices; j++) {
if (!memcmp(&CONFIG.eths[eidx].haddr[0], &ports_eth_addr[j],
ETH_ALEN))
CONFIG.eths[eidx].ifindex = j;
}
/* add to attached devices */
for (j = 0; j < num_devices_attached; j++) {
if (devices_attached[j] == CONFIG.eths[eidx].ifindex) {
break;
}
}
devices_attached[num_devices_attached] = CONFIG.eths[eidx].ifindex;
num_devices_attached++;
fprintf(stderr, "Total number of attached devices: %d\n",
num_devices_attached);
fprintf(stderr, "Interface name: %s\n",
iter_if->ifa_name);
}
iter_if = iter_if->ifa_next;
} while (iter_if != NULL);
freeifaddrs(ifap);
#endif /* !DISABLE_DPDK */
} else if (current_iomodule_func == &netmap_module_func) {
#ifndef DISABLE_NETMAP
struct ifaddrs *ifap;
struct ifaddrs *iter_if;
char *seek;
num_queues = MIN(CONFIG.num_cores, MAX_CPUS);
if (getifaddrs(&ifap) != 0) {
perror("getifaddrs: ");
exit(EXIT_FAILURE);
}
iter_if = ifap;
do {
if (iter_if->ifa_addr->sa_family == AF_INET &&
!set_all_inf &&
(seek=strstr(dev_name_list, iter_if->ifa_name)) != NULL &&
/* check if the interface was not aliased */
*(seek + strlen(iter_if->ifa_name)) != ':') {
struct ifreq ifr;
/* Setting informations */
eidx = CONFIG.eths_num++;
strcpy(CONFIG.eths[eidx].dev_name, iter_if->ifa_name);
strcpy(ifr.ifr_name, iter_if->ifa_name);
/* Create socket */
int sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
if (sock == -1) {
perror("socket");
}
/* getting address */
if (ioctl(sock, SIOCGIFADDR, &ifr) == 0 ) {
struct in_addr sin = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr;
CONFIG.eths[eidx].ip_addr = *(uint32_t *)&sin;
}
if (ioctl(sock, SIOCGIFHWADDR, &ifr) == 0 ) {
for (j = 0; j < ETH_ALEN; j ++) {
CONFIG.eths[eidx].haddr[j] = ifr.ifr_addr.sa_data[j];
}
}
/* Net MASK */
if (ioctl(sock, SIOCGIFNETMASK, &ifr) == 0) {
struct in_addr sin = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr;
CONFIG.eths[eidx].netmask = *(uint32_t *)&sin;
}
close(sock);
#if 0
for (j = 0; j < num_devices; j++) {
if (!memcmp(&CONFIG.eths[eidx].haddr[0], &ports_eth_addr[j],
ETH_ALEN))
CONFIG.eths[eidx].ifindex = ifr.ifr_ifindex;
#endif
CONFIG.eths[eidx].ifindex = eidx;//if_nametoindex(ifr.ifr_name);
TRACE_INFO("Ifindex of interface %s is: %d\n",
ifr.ifr_name, CONFIG.eths[eidx].ifindex);
#if 0
}
#endif
/* add to attached devices */
for (j = 0; j < num_devices_attached; j++) {
if (devices_attached[j] == CONFIG.eths[eidx].ifindex) {
break;
}
}
devices_attached[num_devices_attached] = if_nametoindex(ifr.ifr_name);//CONFIG.eths[eidx].ifindex;
num_devices_attached++;
fprintf(stderr, "Total number of attached devices: %d\n",
num_devices_attached);
fprintf(stderr, "Interface name: %s\n",
iter_if->ifa_name);
}
iter_if = iter_if->ifa_next;
} while (iter_if != NULL);
freeifaddrs(ifap);
#endif /* !DISABLE_NETMAP */
}
return 0;
}
/*----------------------------------------------------------------------------*/
int
SetNetEnv(char *dev_name_list, char *port_stat_list)
{
int eidx = 0;
int i, j;
int set_all_inf = (strncmp(dev_name_list, ALL_STRING, sizeof(ALL_STRING))==0);
TRACE_CONFIG("Loading interface setting\n");
CONFIG.eths = (struct eth_table *)
calloc(MAX_DEVICES, sizeof(struct eth_table));
if (!CONFIG.eths) {
TRACE_ERROR("Can't allocate space for CONFIG.eths\n");
exit(EXIT_FAILURE);
}
if (current_iomodule_func == &ps_module_func) {
#ifndef DISABLE_PSIO
struct ifreq ifr;
/* calculate num_devices now! */
num_devices = ps_list_devices(devices);
if (num_devices == -1) {
perror("ps_list_devices");
exit(EXIT_FAILURE);
}
/* Create socket */
int sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
if (sock == -1) {
TRACE_ERROR("socket");
exit(EXIT_FAILURE);
}
/* To Do: Parse dev_name_list rather than use strstr */
for (i = 0; i < num_devices; i++) {
strcpy(ifr.ifr_name, devices[i].name);
/* getting interface information */
if (ioctl(sock, SIOCGIFFLAGS, &ifr) == 0) {
if (!set_all_inf && strstr(dev_name_list, ifr.ifr_name) == NULL)
continue;
/* Setting informations */
eidx = CONFIG.eths_num++;
strcpy(CONFIG.eths[eidx].dev_name, ifr.ifr_name);
CONFIG.eths[eidx].ifindex = devices[i].ifindex;
/* getting address */
if (ioctl(sock, SIOCGIFADDR, &ifr) == 0 ) {
struct in_addr sin = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr;
CONFIG.eths[eidx].ip_addr = *(uint32_t *)&sin;
}
if (ioctl(sock, SIOCGIFHWADDR, &ifr) == 0 ) {
for (j = 0; j < ETH_ALEN; j ++) {
CONFIG.eths[eidx].haddr[j] = ifr.ifr_addr.sa_data[j];
}
}
/* Net MASK */
if (ioctl(sock, SIOCGIFNETMASK, &ifr) == 0) {
struct in_addr sin = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr;
CONFIG.eths[eidx].netmask = *(uint32_t *)&sin;
}
/* add to attached devices */
for (j = 0; j < num_devices_attached; j++) {
if (devices_attached[j] == devices[i].ifindex) {
break;
}
}
devices_attached[num_devices_attached] = devices[i].ifindex;
num_devices_attached++;
} else {
perror("SIOCGIFFLAGS");
}
}
num_queues = GetNumQueues();
if (num_queues <= 0) {
TRACE_CONFIG("Failed to find NIC queues!\n");
close(sock);
return -1;
}
if (num_queues > num_cpus) {
TRACE_CONFIG("Too many NIC queues available.\n");
close(sock);
return -1;
}
close(sock);
#endif /* !PSIO_MODULE */
} else if (current_iomodule_func == &dpdk_module_func) {
#ifndef DISABLE_DPDK
int cpu = CONFIG.num_cores;
mpz_t _cpumask;
char cpumaskbuf[32] = "";
char mem_channels[8] = "";
char socket_mem_str[32] = "";
// int i;
int ret, socket_mem;
static struct ether_addr ports_eth_addr[RTE_MAX_ETHPORTS];
/* STEP 1: first determine CPU mask */
mpz_init(_cpumask);
if (!mpz_cmp(_cpumask, CONFIG._cpumask)) {
/* get the cpu mask */
for (ret = 0; ret < cpu; ret++)
mpz_setbit(_cpumask, ret);
gmp_sprintf(cpumaskbuf, "%ZX", _cpumask);
} else
gmp_sprintf(cpumaskbuf, "%ZX", CONFIG._cpumask);
mpz_clear(_cpumask);
/* STEP 2: determine memory channels per socket */
/* get the mem channels per socket */
if (CONFIG.num_mem_ch == 0) {
TRACE_ERROR("DPDK module requires # of memory channels "
"per socket parameter!\n");
exit(EXIT_FAILURE);
}
sprintf(mem_channels, "%d", CONFIG.num_mem_ch);
/* STEP 3: determine socket memory */
/* get socket memory threshold (in MB) */
socket_mem =
RTE_ALIGN_CEIL((unsigned long)ceil((CONFIG.num_cores *
(CONFIG.rcvbuf_size +
CONFIG.sndbuf_size +
sizeof(struct tcp_stream) +
sizeof(struct tcp_recv_vars) +
sizeof(struct tcp_send_vars) +
sizeof(struct fragment_ctx)) *
CONFIG.max_concurrency)/RTE_SOCKET_MEM_SHIFT),
RTE_CACHE_LINE_SIZE);
/* initialize the rte env, what a waste of implementation effort! */
int argc = 6;//8;
char *argv[RTE_ARGC_MAX] = {"",
"-c",
cpumaskbuf,
"-n",
mem_channels,
#if 0
"--socket-mem",
socket_mem_str,
#endif
"--proc-type=auto"
};
ret = probe_all_rte_devices(argv, &argc, dev_name_list);
/* STEP 4: build up socket mem parameter */
sprintf(socket_mem_str, "%d", socket_mem);
#if 0
char *smsptr = socket_mem_str + strlen(socket_mem_str);
for (i = 1; i < ret + 1; i++) {
sprintf(smsptr, ",%d", socket_mem);
smsptr += strlen(smsptr);
}
TRACE_DBG("socket_mem: %s\n", socket_mem_str);
#endif
/*
* re-set getopt extern variable optind.
* this issue was a bitch to debug
* rte_eal_init() internally uses getopt() syscall
* mtcp applications that also use an `external' getopt
* will cause a violent crash if optind is not reset to zero
* prior to calling the func below...
* see man getopt(3) for more details
*/
optind = 0;
#ifdef DEBUG
/* print argv's */
for (i = 0; i < argc; i++)
TRACE_INFO("argv[%d]: %s\n", i, argv[i]);
#endif
/* initialize the dpdk eal env */
ret = rte_eal_init(argc, argv);
if (ret < 0) {
TRACE_ERROR("Invalid EAL args!\n");
exit(EXIT_FAILURE);
}
/* give me the count of 'detected' ethernet ports */
#if RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0)
num_devices = rte_eth_dev_count();
#else
num_devices = rte_eth_dev_count_avail();
#endif
if (num_devices == 0) {
TRACE_ERROR("No Ethernet port!\n");
exit(EXIT_FAILURE);
}
/* get mac addr entries of 'detected' dpdk ports */
for (ret = 0; ret < num_devices; ret++)
rte_eth_macaddr_get(ret, &ports_eth_addr[ret]);
num_queues = MIN(CONFIG.num_cores, MAX_CPUS);
struct ifaddrs *ifap;
struct ifaddrs *iter_if;
char *seek;
if (getifaddrs(&ifap) != 0) {
perror("getifaddrs: ");
exit(EXIT_FAILURE);
}
iter_if = ifap;
do {
if (iter_if->ifa_addr && iter_if->ifa_addr->sa_family == AF_INET &&
!set_all_inf &&
(seek=strstr(dev_name_list, iter_if->ifa_name)) != NULL &&
/* check if the interface was not aliased */
*(seek + strlen(iter_if->ifa_name)) != ':') {
struct ifreq ifr;
/* Setting informations */
eidx = CONFIG.eths_num++;
strcpy(CONFIG.eths[eidx].dev_name, iter_if->ifa_name);
strcpy(ifr.ifr_name, iter_if->ifa_name);
/* Create socket */
int sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
if (sock == -1) {
perror("socket");
exit(EXIT_FAILURE);
}
/* getting address */
if (ioctl(sock, SIOCGIFADDR, &ifr) == 0 ) {
struct in_addr sin = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr;
CONFIG.eths[eidx].ip_addr = *(uint32_t *)&sin;
}
if (ioctl(sock, SIOCGIFHWADDR, &ifr) == 0 ) {
for (j = 0; j < ETH_ALEN; j ++) {
CONFIG.eths[eidx].haddr[j] = ifr.ifr_addr.sa_data[j];
}
}
/* Net MASK */
if (ioctl(sock, SIOCGIFNETMASK, &ifr) == 0) {
struct in_addr sin = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr;
CONFIG.eths[eidx].netmask = *(uint32_t *)&sin;
}
close(sock);
for (j = 0; j < num_devices; j++) {
if (!memcmp(&CONFIG.eths[eidx].haddr[0], &ports_eth_addr[j],
ETH_ALEN))
CONFIG.eths[eidx].ifindex = j;
}
/* add to attached devices */
for (j = 0; j < num_devices_attached; j++) {
if (devices_attached[j] == CONFIG.eths[eidx].ifindex) {
break;
}
}
devices_attached[num_devices_attached] = CONFIG.eths[eidx].ifindex;
num_devices_attached++;
fprintf(stderr, "Total number of attached devices: %d\n",
num_devices_attached);
fprintf(stderr, "Interface name: %s\n",
iter_if->ifa_name);
}
iter_if = iter_if->ifa_next;
} while (iter_if != NULL);
freeifaddrs(ifap);
#if 0
/*
* XXX: It seems that there is a bug in the RTE SDK.
* The dynamically allocated rte_argv params are left
* as dangling pointers. Freeing them causes program
* to crash.
*/
/* free up all resources */
for (; rte_argc >= 9; rte_argc--) {
if (rte_argv[rte_argc] != NULL) {
fprintf(stderr, "Cleaning up rte_argv[%d]: %s (%p)\n",
rte_argc, rte_argv[rte_argc], rte_argv[rte_argc]);
free(rte_argv[rte_argc]);
rte_argv[rte_argc] = NULL;
}
}
#endif
/* check if process is primary or secondary */
CONFIG.multi_process_is_master = (eal_proc_type_detect() == RTE_PROC_PRIMARY) ?
1 : 0;
#endif /* !DISABLE_DPDK */
} else if (current_iomodule_func == &netmap_module_func) {
#ifndef DISABLE_NETMAP
struct ifaddrs *ifap;
struct ifaddrs *iter_if;
char *seek;
num_queues = MIN(CONFIG.num_cores, MAX_CPUS);
if (getifaddrs(&ifap) != 0) {
perror("getifaddrs: ");
exit(EXIT_FAILURE);
}
iter_if = ifap;
do {
if (iter_if->ifa_addr && iter_if->ifa_addr->sa_family == AF_INET &&
!set_all_inf &&
(seek=strstr(dev_name_list, iter_if->ifa_name)) != NULL &&
/* check if the interface was not aliased */
*(seek + strlen(iter_if->ifa_name)) != ':') {
struct ifreq ifr;
/* Setting informations */
eidx = CONFIG.eths_num++;
strcpy(CONFIG.eths[eidx].dev_name, iter_if->ifa_name);
strcpy(ifr.ifr_name, iter_if->ifa_name);
/* Create socket */
int sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
if (sock == -1) {
perror("socket");
exit(EXIT_FAILURE);
}
/* getting address */
if (ioctl(sock, SIOCGIFADDR, &ifr) == 0 ) {
struct in_addr sin = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr;
CONFIG.eths[eidx].ip_addr = *(uint32_t *)&sin;
}
if (ioctl(sock, SIOCGIFHWADDR, &ifr) == 0 ) {
for (j = 0; j < ETH_ALEN; j ++) {
CONFIG.eths[eidx].haddr[j] = ifr.ifr_addr.sa_data[j];
}
}
/* Net MASK */
if (ioctl(sock, SIOCGIFNETMASK, &ifr) == 0) {
struct in_addr sin = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr;
CONFIG.eths[eidx].netmask = *(uint32_t *)&sin;
}
close(sock);
#if 0
for (j = 0; j < num_devices; j++) {
if (!memcmp(&CONFIG.eths[eidx].haddr[0], &ports_eth_addr[j],
ETH_ALEN))
CONFIG.eths[eidx].ifindex = ifr.ifr_ifindex;
#endif
CONFIG.eths[eidx].ifindex = eidx;
TRACE_INFO("Ifindex of interface %s is: %d\n",
ifr.ifr_name, CONFIG.eths[eidx].ifindex);
#if 0
}
#endif
/* add to attached devices */
for (j = 0; j < num_devices_attached; j++) {
if (devices_attached[j] == CONFIG.eths[eidx].ifindex) {
break;
}
}
devices_attached[num_devices_attached] = if_nametoindex(ifr.ifr_name);
num_devices_attached++;
fprintf(stderr, "Total number of attached devices: %d\n",
num_devices_attached);
fprintf(stderr, "Interface name: %s\n",
iter_if->ifa_name);
}
iter_if = iter_if->ifa_next;
} while (iter_if != NULL);
freeifaddrs(ifap);
#endif /* !DISABLE_NETMAP */
} else if (current_iomodule_func == &onvm_module_func) {
#ifdef ENABLE_ONVM
int cpu = CONFIG.num_cores;
mpz_t cpumask;
char cpumaskbuf[32];
char mem_channels[8];
char service[6];
char instance[6];
int ret;
mpz_init(cpumask);
/* get the cpu mask */
for (ret = 0; ret < cpu; ret++)
mpz_setbit(cpumask, ret);
gmp_sprintf(cpumaskbuf, "%ZX", cpumask);
mpz_clear(cpumask);
/* get the mem channels per socket */
if (CONFIG.num_mem_ch == 0) {
TRACE_ERROR("DPDK module requires # of memory channels "
"per socket parameter!\n");
exit(EXIT_FAILURE);
}
sprintf(mem_channels, "%d", CONFIG.num_mem_ch);
sprintf(service, "%d", CONFIG.onvm_serv);
sprintf(instance, "%d", CONFIG.onvm_inst);
/* initialize the rte env first, what a waste of implementation effort! */
char *argv[] = {"",
"-c",
cpumaskbuf,
"-n",
mem_channels,
"--proc-type=secondary",
"--",
"-r",
service,
instance,
""
};
const int argc = 10;
/*
* re-set getopt extern variable optind.
* this issue was a bitch to debug
* rte_eal_init() internally uses getopt() syscall
* mtcp applications that also use an `external' getopt
* will cause a violent crash if optind is not reset to zero
* prior to calling the func below...
* see man getopt(3) for more details
*/
optind = 0;
/* initialize the dpdk eal env */
ret = onvm_nflib_init(argc, argv, "mtcp_nf", &CONFIG.nf_info);
if (ret < 0) {
TRACE_ERROR("Invalid EAL args!\n");
exit(EXIT_FAILURE);
}
/* give me the count of 'detected' ethernet ports */
num_devices = ports->num_ports;
if (num_devices == 0) {
TRACE_ERROR("No Ethernet port!\n");
exit(EXIT_FAILURE);
}
num_queues = MIN(CONFIG.num_cores, MAX_CPUS);
struct ifaddrs *ifap;
struct ifaddrs *iter_if;
char *seek;
if (getifaddrs(&ifap) != 0) {
perror("getifaddrs: ");
exit(EXIT_FAILURE);
}
iter_if = ifap;
do {
if (iter_if->ifa_addr && iter_if->ifa_addr->sa_family == AF_INET &&
!set_all_inf &&
(seek=strstr(dev_name_list, iter_if->ifa_name)) != NULL &&
/* check if the interface was not aliased */
*(seek + strlen(iter_if->ifa_name)) != ':') {
struct ifreq ifr;
/* Setting informations */
eidx = CONFIG.eths_num++;
strcpy(CONFIG.eths[eidx].dev_name, iter_if->ifa_name);
strcpy(ifr.ifr_name, iter_if->ifa_name);
/* Create socket */
int sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
if (sock == -1) {
perror("socket");
}
/* getting address */
if (ioctl(sock, SIOCGIFADDR, &ifr) == 0 ) {
struct in_addr sin = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr;
CONFIG.eths[eidx].ip_addr = *(uint32_t *)&sin;
}
for (j = 0; j < ETH_ALEN; j ++) {
CONFIG.eths[eidx].haddr[j] = ports->mac[eidx].addr_bytes[j];
};
/* Net MASK */
if (ioctl(sock, SIOCGIFNETMASK, &ifr) == 0) {
struct in_addr sin = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr;
CONFIG.eths[eidx].netmask = *(uint32_t *)&sin;
}
close(sock);
CONFIG.eths[eidx].ifindex = ports->id[eidx];
devices_attached[num_devices_attached] = CONFIG.eths[eidx].ifindex;
num_devices_attached++;
fprintf(stderr, "Total number of attached devices: %d\n",
num_devices_attached);
fprintf(stderr, "Interface name: %s\n",
iter_if->ifa_name);
}
iter_if = iter_if->ifa_next;
} while (iter_if != NULL);
freeifaddrs(ifap);
#endif /* ENABLE_ONVM */
}
CONFIG.nif_to_eidx = (int*)calloc(MAX_DEVICES, sizeof(int));
if (!CONFIG.nif_to_eidx) {
exit(EXIT_FAILURE);
}
for (i = 0; i < MAX_DEVICES; ++i) {
CONFIG.nif_to_eidx[i] = -1;
}
for (i = 0; i < CONFIG.eths_num; ++i) {
j = CONFIG.eths[i].ifindex;
if (j >= MAX_DEVICES) {
TRACE_ERROR("ifindex of eths_%d exceed the limit: %d\n", i, j);
exit(EXIT_FAILURE);
}
/* the physic port index of the i-th port listed in the config file is j*/
CONFIG.nif_to_eidx[j] = i;
/* finally set the port stats option `on' */
if (strcmp(CONFIG.eths[i].dev_name, port_stat_list) == 0)
CONFIG.eths[i].stat_print = TRUE;
}
return 0;
}
/*----------------------------------------------------------------------------*/
int
main(int argc, char **argv)
{
DIR *dir;
struct dirent *ent;
int fd;
int ret;
uint64_t total_read;
int cores[MAX_CPUS];
int i;
num_cores = GetNumCPUs();
core_limit = num_cores;
if (argc < 2) {
TRACE_ERROR("$%s directory_to_service\n", argv[0]);
return FALSE;
}
/* open the directory to serve */
www_main = argv[1];
dir = opendir(www_main);
if (!dir) {
TRACE_ERROR("Failed to open %s.\n", www_main);
perror("opendir");
return FALSE;
}
for (i = 0; i < argc - 1; i++) {
if (strcmp(argv[i], "-N") == 0) {
core_limit = atoi(argv[i + 1]);
if (core_limit > num_cores) {
TRACE_CONFIG("CPU limit should be smaller than the "
"number of CPUS: %d\n", num_cores);
return FALSE;
}
}
}
nfiles = 0;
while ((ent = readdir(dir)) != NULL) {
if (strcmp(ent->d_name, ".") == 0)
continue;
else if (strcmp(ent->d_name, "..") == 0)
continue;
strcpy(fcache[nfiles].name, ent->d_name);
sprintf(fcache[nfiles].fullname, "%s/%s", www_main, ent->d_name);
fd = open(fcache[nfiles].fullname, O_RDONLY);
if (fd < 0) {
perror("open");
continue;
} else {
fcache[nfiles].size = lseek64(fd, 0, SEEK_END);
lseek64(fd, 0, SEEK_SET);
}
fcache[nfiles].file = (char *)malloc(fcache[nfiles].size);
if (!fcache[nfiles].file) {
TRACE_ERROR("Failed to allocate memory for file %s\n",
fcache[nfiles].name);
perror("malloc");
continue;
}
TRACE_INFO("Reading %s (%lu bytes)\n",
fcache[nfiles].name, fcache[nfiles].size);
total_read = 0;
while (1) {
ret = read(fd, fcache[nfiles].file + total_read,
fcache[nfiles].size - total_read);
if (ret < 0) {
break;
} else if (ret == 0) {
break;
}
total_read += ret;
}
if (total_read < fcache[nfiles].size) {
free(fcache[nfiles].file);
continue;
}
close(fd);
nfiles++;
if (nfiles >= MAX_FILES)
break;
}
finished = 0;
/* initialize mtcp */
ret = mtcp_init("epserver.conf");
if (ret) {
TRACE_ERROR("Failed to initialize mtcp\n");
exit(EXIT_FAILURE);
}
/* register signal handler to mtcp */
mtcp_register_signal(SIGINT, SignalHandler);
TRACE_INFO("Application initialization finished.\n");
for (i = 0; i < core_limit; i++) {
cores[i] = i;
done[i] = FALSE;
if (pthread_create(&app_thread[i],
NULL, RunServerThread, (void *)&cores[i])) {
perror("pthread_create");
TRACE_ERROR("Failed to create server thread.\n");
exit(-1);
}
}
for (i = 0; i < core_limit; i++) {
pthread_join(app_thread[i], NULL);
}
mtcp_destroy();
closedir(dir);
return 0;
}
/*----------------------------------------------------------------------------*/
int
SetInterfaceInfo(char* dev_name_list)
{
struct ifreq ifr;
int eidx = 0;
int i, j;
int set_all_inf = (strncmp(dev_name_list, ALL_STRING, sizeof(ALL_STRING))==0);
TRACE_CONFIG("Loading interface setting\n");
CONFIG.eths = (struct eth_table *)
calloc(MAX_DEVICES, sizeof(struct eth_table));
if (!CONFIG.eths)
exit(EXIT_FAILURE);
if (current_iomodule_func == &ps_module_func) {
/* calculate num_devices now! */
num_devices = ps_list_devices(devices);
if (num_devices == -1) {
perror("ps_list_devices");
exit(EXIT_FAILURE);
}
/* Create socket */
int sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
if (sock == -1) {
perror("socket");
}
/* To Do: Parse dev_name_list rather than use strstr */
for (i = 0; i < num_devices; i++) {
strcpy(ifr.ifr_name, devices[i].name);
/* getting interface information */
if (ioctl(sock, SIOCGIFFLAGS, &ifr) == 0) {
if (!set_all_inf && strstr(dev_name_list, ifr.ifr_name) == NULL)
continue;
/* Setting informations */
eidx = CONFIG.eths_num++;
strcpy(CONFIG.eths[eidx].dev_name, ifr.ifr_name);
CONFIG.eths[eidx].ifindex = devices[i].ifindex;
/* getting address */
if (ioctl(sock, SIOCGIFADDR, &ifr) == 0 ) {
struct in_addr sin = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr;
CONFIG.eths[eidx].ip_addr = *(uint32_t *)&sin;
}
if (ioctl(sock, SIOCGIFHWADDR, &ifr) == 0 ) {
for (j = 0; j < 6; j ++) {
CONFIG.eths[eidx].haddr[j] = ifr.ifr_addr.sa_data[j];
}
}
/* Net MASK */
if (ioctl(sock, SIOCGIFNETMASK, &ifr) == 0) {
struct in_addr sin = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr;
CONFIG.eths[eidx].netmask = *(uint32_t *)&sin;
}
/* add to attached devices */
for (j = 0; j < num_devices_attached; j++) {
if (devices_attached[j] == devices[i].ifindex) {
break;
}
}
devices_attached[num_devices_attached] = devices[i].ifindex;
num_devices_attached++;
} else {
perror("SIOCGIFFLAGS");
}
} // end for: till num_devices
num_queues = GetNumQueues();
if (num_queues <= 0) {
TRACE_CONFIG("Failed to find NIC queues!\n");
return -1;
}
if (num_queues > num_cpus) {
TRACE_CONFIG("Too many NIC queues available.\n");
return -1;
}
} else if (current_iomodule_func == &dpdk_module_func) {
#ifndef DISABLE_DPDK
int cpu = CONFIG.num_cores;
uint32_t cpumask = 0;
char cpumaskbuf[10];
char mem_channels[5];
int ret;
static struct ether_addr ports_eth_addr[RTE_MAX_ETHPORTS];
/* get the cpu mask */
for (ret = 0; ret < cpu; ret++)
cpumask = (cpumask | (1 << ret));
sprintf(cpumaskbuf, "%X", cpumask);
/* get the mem channels per socket */
if (CONFIG.num_mem_ch == 0) {
TRACE_ERROR("DPDK module requires # of memory channels "
"per socket parameter!\n");
exit(EXIT_FAILURE);
}
sprintf(mem_channels, "%d", CONFIG.num_mem_ch);
/* initialize the rte env first, what a waste of implementation effort! */
char *argv[] = {"",
"-c",
cpumaskbuf,
"-n",
mem_channels,
"--proc-type=auto",
""
};
const int argc = 6;
/*
* re-set getopt extern variable optind.
* this issue was a bitch to debug
* rte_eal_init() internally uses getopt() syscall
* mtcp applications that also use an `external' getopt
* will cause a violent crash if optind is not reset to zero
* prior to calling the func below...
* see man getopt(3) for more details
*/
optind = 0;
/* initialize the dpdk eal env */
ret = rte_eal_init(argc, argv);
if (ret < 0)
rte_exit(EXIT_FAILURE, "Invalid EAL args!\n");
/* give me the count of 'detected' ethernet ports */
num_devices = rte_eth_dev_count();
if (num_devices == 0) {
rte_exit(EXIT_FAILURE, "No Ethernet port!\n");
}
/* get mac addr entries of 'detected' dpdk ports */
for (ret = 0; ret < num_devices; ret++)
rte_eth_macaddr_get(ret, &ports_eth_addr[ret]);
num_queues = MIN(CONFIG.num_cores, MAX_CPUS);
struct ifaddrs *ifap;
struct ifaddrs *iter_if;
char *seek;
if (getifaddrs(&ifap) != 0) {
perror("getifaddrs: ");
exit(EXIT_FAILURE);
}
iter_if = ifap;
do {
if (iter_if->ifa_addr->sa_family == AF_INET &&
!set_all_inf &&
(seek=strstr(dev_name_list, iter_if->ifa_name)) != NULL &&
/* check if the interface was not aliased */
*(seek + strlen(iter_if->ifa_name)) != ':') {
struct ifreq ifr;
/* Setting informations */
eidx = CONFIG.eths_num++;
strcpy(CONFIG.eths[eidx].dev_name, iter_if->ifa_name);
strcpy(ifr.ifr_name, iter_if->ifa_name);
/* Create socket */
int sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
if (sock == -1) {
perror("socket");
}
/* getting address */
if (ioctl(sock, SIOCGIFADDR, &ifr) == 0 ) {
struct in_addr sin = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr;
CONFIG.eths[eidx].ip_addr = *(uint32_t *)&sin;
}
if (ioctl(sock, SIOCGIFHWADDR, &ifr) == 0 ) {
for (j = 0; j < 6; j ++) {
CONFIG.eths[eidx].haddr[j] = ifr.ifr_addr.sa_data[j];
}
}
/* Net MASK */
if (ioctl(sock, SIOCGIFNETMASK, &ifr) == 0) {
struct in_addr sin = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr;
CONFIG.eths[eidx].netmask = *(uint32_t *)&sin;
}
close(sock);
for (j = 0; j < num_devices; j++) {
if (!memcmp(&CONFIG.eths[eidx].haddr[0], &ports_eth_addr[j],
ETH_ALEN))
CONFIG.eths[eidx].ifindex = j;
}
/* add to attached devices */
for (j = 0; j < num_devices_attached; j++) {
if (devices_attached[j] == CONFIG.eths[eidx].ifindex) {
break;
}
}
devices_attached[num_devices_attached] = CONFIG.eths[eidx].ifindex;
num_devices_attached++;
fprintf(stderr, "Total number of attached devices: %d\n",
num_devices_attached);
fprintf(stderr, "Interface name: %s\n",
iter_if->ifa_name);
}
iter_if = iter_if->ifa_next;
} while (iter_if != NULL);
freeifaddrs(ifap);
#endif /* !DISABLE_DPDK */
} else if (current_iomodule_func == &tuntap_module_func) {
printf("Trying to configure tuntap device\n");
#ifndef DISABLE_TUNTAP
/* Let the great duplication of the code begin!!! */
// FIXME: Do I need this?
num_queues = MIN(CONFIG.num_cores, MAX_CPUS);
// FIXME: Currently tuntap will have only one queue
assert(num_queues == 1);
// Initializing the tuntap device as we want to read
// their IP addresses and other settings
// FIXME: Do this once per device in the list devices
init_tap_network("tap0", "192.168.123.100", "255.255.255.0");
struct ifaddrs *ifap;
struct ifaddrs *iter_if;
char *seek;
if (getifaddrs(&ifap) != 0) {
perror("getifaddrs: ");
exit(EXIT_FAILURE);
}
int if_count = 0;
int if_count_used = 0;
iter_if = ifap;
do {
if (iter_if->ifa_addr->sa_family == AF_INET &&
!set_all_inf &&
(seek=strstr(dev_name_list, iter_if->ifa_name)) != NULL &&
/* check if the interface was not aliased */
*(seek + strlen(iter_if->ifa_name)) != ':') {
struct ifreq ifr;
/* Setting informations */
eidx = CONFIG.eths_num++;
CONFIG.eths[eidx].ifindex = if_count_used;
++if_count_used;
strcpy(CONFIG.eths[eidx].dev_name, iter_if->ifa_name);
strcpy(ifr.ifr_name, iter_if->ifa_name);
/* Create socket */
int sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
if (sock == -1) {
perror("socket");
}
/* getting address */
if (ioctl(sock, SIOCGIFADDR, &ifr) == 0 ) {
//struct in_addr sin = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr;
in_addr_t addr = inet_addr("192.168.123.1");
//CONFIG.eths[eidx].ip_addr = *(uint32_t *)&sin;
CONFIG.eths[eidx].ip_addr = addr;
printf("IP addr: %"PRIx32" \n", CONFIG.eths[eidx].ip_addr);
}
if (ioctl(sock, SIOCGIFHWADDR, &ifr) == 0 ) {
for (j = 0; j < 6; j ++) {
CONFIG.eths[eidx].haddr[j] = ifr.ifr_addr.sa_data[j];
}
// FIXME: Ugly hack to get a different MAC address by subtracting
// one from the gateway's MAC address
CONFIG.eths[eidx].haddr[5] = CONFIG.eths[eidx].haddr[5] - 1;
}
/* Net MASK */
if (ioctl(sock, SIOCGIFNETMASK, &ifr) == 0) {
struct in_addr sin = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr;
CONFIG.eths[eidx].netmask = *(uint32_t *)&sin;
printf("netmask: %"PRIx32" \n", CONFIG.eths[eidx].netmask);
}
close(sock);
/* add to attached devices */
for (j = 0; j < num_devices_attached; j++) {
if (devices_attached[j] == CONFIG.eths[eidx].ifindex) {
break;
}
}
devices_attached[num_devices_attached] = CONFIG.eths[eidx].ifindex;
num_devices_attached++;
fprintf(stderr, "Total number of attached devices: %d\n",
num_devices_attached);
fprintf(stderr, "Interface name: %s\n",
iter_if->ifa_name);
} // end if: Valid device found
else {
fprintf(stderr, "INFO: Ignoring Interface name: %s\n",
iter_if->ifa_name);
} // end else:
iter_if = iter_if->ifa_next;
++if_count;
} while (iter_if != NULL); // for each interface in local system
freeifaddrs(ifap);
fprintf(stderr, "Total number of attached devices: %d\n",
num_devices_attached);
#endif // DISABLE_TUNTAP
} else {
printf("ERROR: This IO Module is not yet supported by %s:%s\n",
__FILE__, __FUNCTION__);
exit(EXIT_FAILURE);
}
return 0;
} // end function: SetInterfaceInfo
/*----------------------------------------------------------------------------*/
int
main(int argc, char **argv)
{
DIR *dir;
struct dirent *ent;
int fd;
int ret;
uint64_t total_read;
struct mtcp_conf mcfg;
int cores[MAX_CPUS];
int process_cpu;
int i, o;
num_cores = GetNumCPUs();
core_limit = num_cores;
process_cpu = -1;
dir = NULL;
if (argc < 2) {
TRACE_CONFIG("$%s directory_to_service\n", argv[0]);
return FALSE;
}
while (-1 != (o = getopt(argc, argv, "N:f:p:c:h"))) {
switch (o) {
case 'p':
/* open the directory to serve */
www_main = optarg;
dir = opendir(www_main);
if (!dir) {
TRACE_CONFIG("Failed to open %s.\n", www_main);
perror("opendir");
return FALSE;
}
break;
case 'N':
core_limit = atoi(optarg);
if (core_limit > num_cores) {
TRACE_CONFIG("CPU limit should be smaller than the "
"number of CPUs: %d\n", num_cores);
return FALSE;
}
/**
* it is important that core limit is set
* before mtcp_init() is called. You can
* not set core_limit after mtcp_init()
*/
mtcp_getconf(&mcfg);
mcfg.num_cores = core_limit;
mtcp_setconf(&mcfg);
break;
case 'f':
conf_file = optarg;
break;
case 'c':
process_cpu = atoi(optarg);
if (process_cpu > core_limit) {
TRACE_CONFIG("Starting CPU is way off limits!\n");
return FALSE;
}
break;
case 'h':
printHelp(argv[0]);
break;
}
}
if (dir == NULL) {
TRACE_CONFIG("You did not pass a valid www_path!\n");
exit(EXIT_FAILURE);
}
nfiles = 0;
while ((ent = readdir(dir)) != NULL) {
if (strcmp(ent->d_name, ".") == 0)
continue;
else if (strcmp(ent->d_name, "..") == 0)
continue;
strcpy(fcache[nfiles].name, ent->d_name);
sprintf(fcache[nfiles].fullname, "%s/%s", www_main, ent->d_name);
fd = open(fcache[nfiles].fullname, O_RDONLY);
if (fd < 0) {
perror("open");
continue;
} else {
fcache[nfiles].size = lseek64(fd, 0, SEEK_END);
lseek64(fd, 0, SEEK_SET);
}
fcache[nfiles].file = (char *)malloc(fcache[nfiles].size);
if (!fcache[nfiles].file) {
TRACE_CONFIG("Failed to allocate memory for file %s\n",
fcache[nfiles].name);
perror("malloc");
continue;
}
TRACE_INFO("Reading %s (%lu bytes)\n",
fcache[nfiles].name, fcache[nfiles].size);
total_read = 0;
while (1) {
ret = read(fd, fcache[nfiles].file + total_read,
fcache[nfiles].size - total_read);
if (ret < 0) {
break;
} else if (ret == 0) {
break;
}
total_read += ret;
}
if (total_read < fcache[nfiles].size) {
free(fcache[nfiles].file);
continue;
}
close(fd);
nfiles++;
if (nfiles >= MAX_FILES)
break;
}
finished = 0;
/* initialize mtcp */
if (conf_file == NULL) {
TRACE_CONFIG("You forgot to pass the mTCP startup config file!\n");
exit(EXIT_FAILURE);
}
ret = mtcp_init(conf_file);
if (ret) {
TRACE_CONFIG("Failed to initialize mtcp\n");
exit(EXIT_FAILURE);
}
/* register signal handler to mtcp */
mtcp_register_signal(SIGINT, SignalHandler);
TRACE_INFO("Application initialization finished.\n");
for (i = ((process_cpu == -1) ? 0 : process_cpu); i < core_limit; i++) {
cores[i] = i;
done[i] = FALSE;
if (pthread_create(&app_thread[i],
NULL, RunServerThread, (void *)&cores[i])) {
perror("pthread_create");
TRACE_CONFIG("Failed to create server thread.\n");
exit(EXIT_FAILURE);
}
if (process_cpu != -1)
break;
}
for (i = ((process_cpu == -1) ? 0 : process_cpu); i < core_limit; i++) {
pthread_join(app_thread[i], NULL);
if (process_cpu != -1)
break;
}
mtcp_destroy();
closedir(dir);
return 0;
}
