int main()
{
int ret;
ret = odp_rpc_server_start();
if (ret) {
fprintf(stderr, "[RPC] Error: Failed to start server\n");
exit(EXIT_FAILURE);
}
if ( __k1_get_cluster_id() == 128 ) {
printf("Spawning clusters\n");
{
static char const * _argv[] = {
"odp_ipsec.kelf",
"-i", "e0:loop,e1:loop",
"-r", "192.168.111.2/32:e0:00.07.43.30.4a.70",
"-r", "192.168.222.2/32:e1:00.07.43.30.4a.78",
"-p", "192.168.111.0/24:192.168.222.0/24:out:both",
"-e", "192.168.111.2:192.168.222.2:aesgcm:201:656c8523255ccc23a66c1917aa0cf309",
"-a", "192.168.111.2:192.168.222.2:aesgcm:200:656c8523255ccc23a66c1917aa0cf309",
"-p", "192.168.222.0/24:192.168.111.0/24:in:both",
"-e", "192.168.222.2:192.168.111.2:aesgcm:301:656c8523255ccc23a66c1917aa0cf309",
"-a", "192.168.222.2:192.168.111.2:aesgcm:300:656c8523255ccc23a66c1917aa0cf309",
"-c", "14", "-m", "ASYNC_IN_PLACE", NULL
};
for (int i = 0; i < 16; i++) {
if ( i % 4 == 0 ) continue;
if ( i % 4 == 3 ) continue;
boot_cluster(i, _argv[0], _argv);
}
}
{
static char const * _argv[] = {
"odp_generator.kelf",
"-I", "e0:loop:nofree",
"--srcmac", "08:00:27:76:b5:e0",
"--dstmac", "00:00:00:00:80:01",
"--srcip", "192.168.111.2",
"--dstip", "192.168.222.2",
"-m", "u",
"-i", "0", "-w", "2", "-p", "256", NULL
};
for (int i = 0; i < 16; i += 4) {
if ( i % 4 == 1 ) continue;
if ( i % 4 == 2 ) continue;
if ( i % 4 == 3 ) continue;
boot_cluster(i, _argv[0], _argv);
}
}
printf("Cluster booted\n");
}
join_clusters(NULL);
return 0;
}
void mppa_barrier_wait(barrier_t *barrier) {
int status;
long long dummy;
if(barrier->mode == BARRIER_MASTER) {
dummy = -1;
long long match;
status = mppa_read(barrier->sync_fd_master, &match, sizeof(match));
assert(status == sizeof(match));
status = mppa_write(barrier->sync_fd_slave, &dummy, sizeof(long long));
assert(status == sizeof(long long));
}
else {
dummy = 0;
long long mask;
mask = 0;
mask |= 1 << __k1_get_cluster_id();
status = mppa_write(barrier->sync_fd_master, &mask, sizeof(mask));
assert(status == sizeof(mask));
status = mppa_read(barrier->sync_fd_slave, &dummy, sizeof(long long));
assert(status == sizeof(long long));
}
}
int boot_cluster(int clus_id, const char bin_file[], const char * argv[] ) {
if (__k1_get_cluster_id() != 128)
return -1;
struct clus_bin_boot *clus = &clus_bin_boots[clus_id];
has_booted = 1;
if (clus->status != STATE_OFF)
return -1;
#ifdef VERBOSE
printf("[BOOT] Spawning cluster %d with binary %s\n",
clus_id, bin_file);
#endif
clus->bin = strdup(bin_file);
clus->id = clus_id;
clus->argv[0] = clus->bin;
clus->argc = 1;
clus->pid =
mppa_power_base_spawn(clus->id,
bin_file,
argv,
NULL,
MPPA_POWER_SHUFFLING_DISABLED);
if (clus->pid < 0) {
fprintf(stderr, "Failed to spawn cluster %d\n", clus_id);
return -1;
}
clus->status = STATE_ON;
return 0;
}
int boot_clusters(int argc, char * const argv[])
{
unsigned int i;
int opt;
if (__k1_get_cluster_id() != 128)
return -1;
while ((opt = getopt(argc, argv, "c:a:")) != -1) {
switch (opt) {
case 'c':
{
struct clus_bin_boot *clus =
&clus_bin_boots[clus_count];
clus->bin = strdup(optarg);
clus->id = clus_count;
clus->argv[0] = clus->bin;
clus->argc = 1;
clus_count++;
}
break;
case 'a':
{
char *pch = strtok(strdup(optarg), " ");
while ( pch != NULL ) {
struct clus_bin_boot *clus =
&clus_bin_boots[clus_count - 1];
clus->argv[clus->argc] = pch;
clus->argc++;
pch = strtok(NULL, " ");
}
}
break;
default: /* '?' */
fprintf(stderr, "Wrong arguments for boot\n");
return -1;
}
}
for (i = 0; i < clus_count; i++) {
struct clus_bin_boot *clus = &clus_bin_boots[i];
clus->argv[clus->argc] = NULL;
if (boot_cluster(i, clus->argv[0], clus->argv))
return -1;
}
return 0;
}
portal_t *mppa_create_write_portal (char *path, void* buffer, unsigned long buffer_size, int receiver_rank) {
portal_t *ret = (portal_t*) malloc (sizeof(portal_t));
ret->file_descriptor = mppa_open(path, O_WRONLY);
assert(ret->file_descriptor != -1);
// Tell mppa_io_write to wait for resources when sending a asynchronous message
assert(mppa_ioctl(ret->file_descriptor, MPPA_TX_WAIT_RESOURCE_ON) == 0);
// Select the DMA interface according to the receiver's rank.
// This is only possible on the IO-node!
if (__k1_get_cluster_id() == 128)
assert(mppa_ioctl(ret->file_descriptor, MPPA_TX_SET_IFACE, receiver_rank % 4) == 0);
// We need to initialize an aiocb for asynchronous writes.
// It seems that the buffer and buffer size parameters are not important here,
// because we're going to specify them with mppa_aiocb_set_pwrite()
// before calling mppa_aio_write()
assert(mppa_aiocb_ctor(&ret->aiocb, ret->file_descriptor, buffer, buffer_size) == &ret->aiocb);
return ret;
}
/**
* ODP packet example main function
*/
int main(int argc, char * argv[])
{
odph_linux_pthread_t thread_tbl[MAX_WORKERS];
odp_pool_t pool;
int num_workers;
int i;
odp_shm_t shm;
odp_cpumask_t cpumask;
char cpumaskstr[ODP_CPUMASK_STR_SIZE];
odp_pool_param_t params;
odp_timer_pool_param_t tparams;
odp_timer_pool_t tp;
odp_pool_t tmop;
/* Init ODP before calling anything else */
if (odp_init_global(NULL, NULL)) {
EXAMPLE_ERR("Error: ODP global init failed.\n");
exit(EXIT_FAILURE);
}
if (odp_init_local(ODP_THREAD_CONTROL)) {
EXAMPLE_ERR("Error: ODP local init failed.\n");
exit(EXIT_FAILURE);
}
my_sleep(1 + __k1_get_cluster_id() / 4);
/* init counters */
odp_atomic_init_u64(&counters.seq, 0);
odp_atomic_init_u64(&counters.ip, 0);
odp_atomic_init_u64(&counters.udp, 0);
odp_atomic_init_u64(&counters.icmp, 0);
odp_atomic_init_u64(&counters.cnt, 0);
/* Reserve memory for args from shared mem */
shm = odp_shm_reserve("shm_args", sizeof(args_t),
ODP_CACHE_LINE_SIZE, 0);
args = odp_shm_addr(shm);
if (args == NULL) {
EXAMPLE_ERR("Error: shared mem alloc failed.\n");
exit(EXIT_FAILURE);
}
memset(args, 0, sizeof(*args));
/* Parse and store the application arguments */
parse_args(argc, argv, &args->appl);
/* Print both system and application information */
print_info(NO_PATH(argv[0]), &args->appl);
/* Default to system CPU count unless user specified */
num_workers = MAX_WORKERS;
if (args->appl.cpu_count)
num_workers = args->appl.cpu_count;
num_workers = odp_cpumask_default_worker(&cpumask, num_workers);
if (args->appl.mask) {
odp_cpumask_from_str(&cpumask, args->appl.mask);
num_workers = odp_cpumask_count(&cpumask);
}
(void)odp_cpumask_to_str(&cpumask, cpumaskstr, sizeof(cpumaskstr));
printf("num worker threads: %i\n", num_workers);
printf("first CPU: %i\n", odp_cpumask_first(&cpumask));
printf("cpu mask: %s\n", cpumaskstr);
/* ping mode need two workers */
if (args->appl.mode == APPL_MODE_PING) {
if (num_workers < 2) {
EXAMPLE_ERR("Need at least two worker threads\n");
exit(EXIT_FAILURE);
} else {
num_workers = 2;
}
}
/* Create packet pool */
odp_pool_param_init(¶ms);
params.pkt.seg_len = SHM_PKT_POOL_BUF_SIZE;
params.pkt.len = SHM_PKT_POOL_BUF_SIZE;
params.pkt.num = SHM_PKT_POOL_SIZE/SHM_PKT_POOL_BUF_SIZE;
params.type = ODP_POOL_PACKET;
pool = odp_pool_create("packet_pool", ¶ms);
if (pool == ODP_POOL_INVALID) {
EXAMPLE_ERR("Error: packet pool create failed.\n");
exit(EXIT_FAILURE);
}
odp_pool_print(pool);
/* Create timer pool */
tparams.res_ns = 1 * ODP_TIME_MSEC_IN_NS;
tparams.min_tmo = 0;
tparams.max_tmo = 10000 * ODP_TIME_SEC_IN_NS;
tparams.num_timers = num_workers; /* One timer per worker */
tparams.priv = 0; /* Shared */
tparams.clk_src = ODP_CLOCK_CPU;
tp = odp_timer_pool_create("timer_pool", &tparams);
if (tp == ODP_TIMER_POOL_INVALID) {
EXAMPLE_ERR("Timer pool create failed.\n");
exit(EXIT_FAILURE);
}
odp_timer_pool_start();
/* Create timeout pool */
memset(¶ms, 0, sizeof(params));
params.tmo.num = tparams.num_timers; /* One timeout per timer */
params.type = ODP_POOL_TIMEOUT;
tmop = odp_pool_create("timeout_pool", ¶ms);
if (pool == ODP_POOL_INVALID) {
EXAMPLE_ERR("Error: packet pool create failed.\n");
exit(EXIT_FAILURE);
}
for (i = 0; i < args->appl.if_count; ++i)
create_pktio(args->appl.if_names[i], pool);
/* Create and init worker threads */
memset(thread_tbl, 0, sizeof(thread_tbl));
if (args->appl.mode == APPL_MODE_PING) {
odp_cpumask_t cpu_mask;
odp_queue_t tq;
int cpu_first, cpu_next;
odp_cpumask_zero(&cpu_mask);
cpu_first = odp_cpumask_first(&cpumask);
odp_cpumask_set(&cpu_mask, cpu_first);
tq = odp_queue_create("", ODP_QUEUE_TYPE_POLL, NULL);
if (tq == ODP_QUEUE_INVALID)
abort();
args->thread[1].pktio_dev = args->appl.if_names[0];
args->thread[1].pool = pool;
args->thread[1].tp = tp;
args->thread[1].tq = tq;
args->thread[1].tim = odp_timer_alloc(tp, tq, NULL);
if (args->thread[1].tim == ODP_TIMER_INVALID)
abort();
args->thread[1].tmo_ev = odp_timeout_alloc(tmop);
if (args->thread[1].tmo_ev == ODP_TIMEOUT_INVALID)
abort();
args->thread[1].mode = args->appl.mode;
odph_linux_pthread_create(&thread_tbl[1], &cpu_mask,
gen_recv_thread, &args->thread[1],
ODP_THREAD_WORKER);
tq = odp_queue_create("", ODP_QUEUE_TYPE_POLL, NULL);
if (tq == ODP_QUEUE_INVALID)
abort();
args->thread[0].pktio_dev = args->appl.if_names[0];
args->thread[0].pool = pool;
args->thread[0].tp = tp;
args->thread[0].tq = tq;
args->thread[0].tim = odp_timer_alloc(tp, tq, NULL);
if (args->thread[0].tim == ODP_TIMER_INVALID)
abort();
args->thread[0].tmo_ev = odp_timeout_alloc(tmop);
if (args->thread[0].tmo_ev == ODP_TIMEOUT_INVALID)
abort();
args->thread[0].mode = args->appl.mode;
cpu_next = odp_cpumask_next(&cpumask, cpu_first);
odp_cpumask_zero(&cpu_mask);
odp_cpumask_set(&cpu_mask, cpu_next);
odph_linux_pthread_create(&thread_tbl[0], &cpu_mask,
gen_send_thread, &args->thread[0],
ODP_THREAD_WORKER);
} else {
int cpu = odp_cpumask_first(&cpumask);
for (i = 0; i < num_workers; ++i) {
odp_cpumask_t thd_mask;
void *(*thr_run_func) (void *);
int if_idx;
odp_queue_t tq;
if_idx = i % args->appl.if_count;
args->thread[i].pktio_dev = args->appl.if_names[if_idx];
tq = odp_queue_create("", ODP_QUEUE_TYPE_POLL, NULL);
if (tq == ODP_QUEUE_INVALID)
abort();
args->thread[i].pool = pool;
args->thread[i].tp = tp;
args->thread[i].tq = tq;
args->thread[i].tim = odp_timer_alloc(tp, tq, NULL);
if (args->thread[i].tim == ODP_TIMER_INVALID)
abort();
args->thread[i].tmo_ev = odp_timeout_alloc(tmop);
if (args->thread[i].tmo_ev == ODP_TIMEOUT_INVALID)
abort();
args->thread[i].mode = args->appl.mode;
if (args->appl.mode == APPL_MODE_UDP) {
thr_run_func = gen_send_thread;
} else if (args->appl.mode == APPL_MODE_RCV) {
thr_run_func = gen_recv_thread;
} else {
EXAMPLE_ERR("ERR MODE\n");
exit(EXIT_FAILURE);
}
/*
* Create threads one-by-one instead of all-at-once,
* because each thread might get different arguments.
* Calls odp_thread_create(cpu) for each thread
*/
odp_cpumask_zero(&thd_mask);
odp_cpumask_set(&thd_mask, cpu);
odph_linux_pthread_create(&thread_tbl[i],
&thd_mask,
thr_run_func,
&args->thread[i],
ODP_THREAD_WORKER);
cpu = odp_cpumask_next(&cpumask, cpu);
}
}
print_global_stats(num_workers);
/* Master thread waits for other threads to exit */
odph_linux_pthread_join(thread_tbl, num_workers);
free(args->appl.if_names);
free(args->appl.if_str);
printf("Exit\n\n");
return 0;
}
int main(int argc, char *const argv[])
{
int ret;
unsigned clusters = 0;
unsigned n_clusters = 0;
int opt;
while ((opt = getopt(argc, argv, "c:h")) != -1) {
switch (opt) {
case 'c':
{
unsigned mask = 1 << atoi(optarg);
if ((clusters & mask) == 0)
n_clusters ++;
clusters |= mask;
}
break;
case 'h':
printf("Usage: %s [ -c <clus_id> -c <clus_id> -c ... ]", argv[0]);
exit(0);
break;
default: /* '?' */
fprintf(stderr, "Wrong arguments\n");
return -1;
}
}
if (clusters == 0) {
clusters = 0xffff;
n_clusters = 16;
}
ret = pcie_init(MPPA_PCIE_ETH_IF_MAX, 0);
if (ret != 0) {
fprintf(stderr, "Failed to initialize PCIe eth interface\n");
exit(1);
}
ret = odp_rpc_server_start();
if (ret) {
fprintf(stderr, "[RPC] Error: Failed to start server\n");
exit(EXIT_FAILURE);
}
if ( __k1_get_cluster_id() == 128 ) {
printf("Spawning clusters\n");
{
static char const * _argv[] = {
"odp_l2fwd.kelf",
"-i", "p0p0:tags=120,p1p0:tags=120",
"-m", "0",
"-s", "0",
"-c", "10", NULL
};
while(clusters) {
int clus_id = __builtin_k1_ctz(clusters);
clusters &= ~ (1 << clus_id);
boot_cluster(clus_id, _argv[0], _argv);
}
}
printf("Cluster booted\n");
}
join_clusters(NULL);
return 0;
}
int main(int argc, char *const argv[])
{
int ret;
unsigned n_clusters = 1;
int opt;
while ((opt = getopt(argc, argv, "c:h")) != -1) {
switch (opt) {
case 'c':
n_clusters = atoi(optarg);
break;
case 'h':
printf("Usage: %s [ -c <n_clusters> (number of l2fwd clusters, 1..14) ]", argv[0]);
exit(0);
break;
default: /* '?' */
fprintf(stderr, "Wrong arguments\n");
return -1;
}
}
if (!n_clusters) n_clusters = 1;
if (n_clusters > 14) n_clusters = 14;
ret = pcie_init(MPPA_PCIE_ETH_IF_MAX, 0);
if (ret != 0) {
fprintf(stderr, "Failed to initialize PCIe eth interface\n");
exit(1);
}
ret = odp_rpc_server_start();
if (ret) {
fprintf(stderr, "[RPC] Error: Failed to start server\n");
exit(EXIT_FAILURE);
}
if ( __k1_get_cluster_id() == 128 ) {
printf("Spawning clusters\n");
{
static char const * _argv[] = {
"odp_l2fwd.kelf",
"-i", "e0:tags=120:min_payload=48:max_payload=48,e1:tags=120:min_payload=48:max_payload=48",
"-m", "0",
"-s", "0",
"-a", "2",
//"-S",
//"-t", "30",
"-c", "8", NULL
};
for (unsigned i = 0; i < n_clusters; i++)
boot_cluster(i, _argv[0], _argv);
}
if (1)
{
static char const * _argv[] = {
"odp_generator.kelf",
"-I", "e0:nofree", // generates traffic on eth0,
"--srcmac", "08:00:27:76:b5:e0",
"--dstmac", "00:00:00:00:80:01",
"--srcip", "192.168.111.2",
"--dstip", "192.168.222.2",
"-m", "u", // UDP mode
"-i", "0", // interval between sends
"-w", "1", // worker generating traffic per cluster
"-P", "64", // total packet length 64B
NULL
};
boot_cluster(14, _argv[0], _argv);
}
if (1)
{
static char const * _argv[] = {
"odp_generator.kelf",
"-I", "e1:nofree", // generates traffic on eth1,
"--srcmac", "08:00:27:76:b5:e1",
"--dstmac", "00:00:00:00:80:01",
"--srcip", "192.168.111.2",
"--dstip", "192.168.222.2",
"-m", "u", // UDP mode
"-i", "0", // interval between sends
"-w", "1", // worker generating traffic per cluster
"-P", "64", // total packet length 64B
NULL
};
boot_cluster(15, _argv[0], _argv);
}
printf("Clusters booted\n");
}
join_clusters(NULL);
return 0;
}
