char *RunModeTileGetPipelineConfig(const char *custom_mode) {
intmax_t pipelines;
intmax_t detect_per_pipe;
intmax_t value = 0;
char *s;
if (custom_mode != NULL) {
return custom_mode;
}
char *runmode = NULL;
if (ConfGet("runmode", &runmode) == 1) {
if (strcmp(runmode, "workers") == 0) {
/* Available cpus */
cpu_set_t cpus;
tmc_cpus_get_dataplane_cpus(&cpus);
uint16_t ncpus = tmc_cpus_count(&cpus);
TileNumPipelines = ncpus - 1;
return runmode;
}
}
if (ConfGetInt("tile.pipelines", &pipelines) == 1) {
TileNumPipelines = pipelines;
} else {
TileNumPipelines = DFLT_TILERA_PIPELINES;
}
SCLogInfo("%d Tilera pipelines", TileNumPipelines);
if (ConfGetInt("tile.detect-per-pipeline", &detect_per_pipe) == 1) {
TileDetectThreadPerPipeline = detect_per_pipe;
} else {
TileDetectThreadPerPipeline = DFLT_DETECT_THREADS_PER_PIPELINE;
}
if ((ConfGetInt("mpipe.poll", &value)) == 1) {
/* only 1 and 2 are permitted */
if ((value >= 1) && (value <= 2)) {
TileNumPipelinesPerRx = (unsigned int) value;
} else {
SCLogError(SC_ERR_FATAL, "Illegal mpipe.poll value.");
}
}
if (ConfGet("tile.queue", &s) == 1) {
if (strcmp(s, "simple") == 0) {
queue_type = simple;
} else if (strcmp(s, "tmc") == 0) {
queue_type = tmc;
}
}
SCLogInfo("%d detect threads per pipeline", TileDetectThreadPerPipeline);
SCLogInfo("%d utilized dataplane tiles", (TILES_PER_PIPELINE * TileNumPipelines) + (TileNumPipelines / 2));
SCLogInfo("%s queueing between tiles", (queue_type == simple) ? "simple" : "tmc");
return NULL;
}
/**
* \brief RunModeIdsTileMpipeWorkers set up the following thread packet handlers:
* - Receive thread (from iface pcap)
* - Decode thread
* - Stream thread
* - Detect: If we have only 1 cpu, it will setup one Detect thread
* If we have more than one, it will setup num_cpus - 1
* starting from the second cpu available.
* - Respond/Reject thread
* - Outputs thread
* By default the threads will use the first cpu available
* except the Detection threads if we have more than one cpu
*
* \param de_ctx pointer to the Detection Engine
* \param iface pointer to the name of the interface from which we will
* fetch the packets
* \retval 0 if all goes well. (If any problem is detected the engine will
* exit())
*/
int RunModeIdsTileMpipeWorkers(DetectEngineCtx *de_ctx) {
SCEnter();
char tname[32];
char *thread_name;
TmModule *tm_module;
int pipe;
char *detectmode = NULL;
int pool_detect_threads = 0;
extern TmEcode ReceiveMpipeInit(void); // move this
if (ConfGet("tile.detect", &detectmode) == 1) {
if (detectmode) {
SCLogInfo("DEBUG: detectmode %s", detectmode);
if (strcmp(detectmode, "pooled") == 0) {
pool_detect_threads = 1;
}
}
}
RunModeTileMpipeMapCores();
RunModeInitialize();
/* Available cpus */
cpu_set_t cpus;
tmc_cpus_get_dataplane_cpus(&cpus);
uint16_t ncpus = tmc_cpus_count(&cpus);
TimeModeSetLive();
int pipe_max = ncpus - 1;
TileNumPipelines = pipe_max;
TileNumPipelinesPerRx = 1;
ReceiveMpipeInit();
char *mpipe_dev = NULL;
int nlive = LiveGetDeviceCount();
if (nlive > 0) {
SCLogInfo("Using %d live device(s).", nlive);
/*mpipe_dev = LiveGetDevice(0);*/
} else {
/*
* Attempt to get interface from config file
* overrides -i from command line.
*/
if (ConfGet("mpipe.interface", &mpipe_dev) == 0) {
if (ConfGet("mpipe.single_mpipe_dev", &mpipe_dev) == 0) {
SCLogError(SC_ERR_RUNMODE, "Failed retrieving "
"mpipe.single_mpipe_dev from Conf");
exit(EXIT_FAILURE);
}
}
}
/*
* Careful. All of the pickup_queues must be created
* prior to building to pipeline so that the queues
* are adjacent in the lookup table. This lets the
* demux2 queue handler work.
*/
for (pipe = 0; pipe < pipe_max; pipe++) {
sprintf(pickup_queue[pipe], "pickup-queue%d", pipe);
if (TmqCreateQueue(pickup_queue[pipe]) == NULL) {
SCLogError(SC_ERR_RUNMODE, "Could not create pickup queue");
exit(EXIT_FAILURE);
}
}
for (pipe = 0; pipe < pipe_max; pipe++) {
char *mpipe_devc;
/* HACK: Receive Threads are shared between pairs of
* pipelines. So for every other pipeline create two
* queues and spawn only one thread.
*/
if (nlive > 0) {
mpipe_devc = SCStrdup("multi");
} else {
mpipe_devc = SCStrdup(mpipe_dev);
}
snprintf(tname, sizeof(tname), "Worker%d", pipe+1);
thread_name = SCStrdup(tname);
/* create the threads */
ThreadVars *tv_worker =
TmThreadCreatePacketHandler(thread_name,
"packetpool", "packetpool",
"packetpool", "packetpool",
"pktacqloop");
if (tv_worker == NULL) {
printf("ERROR: TmThreadsCreate failed\n");
exit(EXIT_FAILURE);
}
tm_module = TmModuleGetByName("ReceiveMpipe");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName failed for ReceiveMpipe\n");
exit(EXIT_FAILURE);
}
TmSlotSetFuncAppend(tv_worker, tm_module, (void *)mpipe_devc);
/* set affinity for worker */
TmThreadSetCPUAffinity(tv_worker, 1+pipe);
SCLogInfo("Thread %s pipe_max %d pipe %d cpu %d",
thread_name, pipe_max, pipe,
1+pipe);
tm_module = TmModuleGetByName("DecodeMpipe");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName DecodeMpipe failed\n");
exit(EXIT_FAILURE);
}
TmSlotSetFuncAppend(tv_worker,tm_module,NULL);
tm_module = TmModuleGetByName("StreamTcp");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName StreamTcp failed\n");
exit(EXIT_FAILURE);
}
TmSlotSetFuncAppend(tv_worker,tm_module,NULL);
tm_module = TmModuleGetByName("Detect");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName Detect failed\n");
exit(EXIT_FAILURE);
}
TmSlotSetFuncAppend(tv_worker,tm_module,(void *)de_ctx);
tm_module = TmModuleGetByName("RespondReject");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for RespondReject failed\n");
exit(EXIT_FAILURE);
}
TmSlotSetFuncAppend(tv_worker,tm_module,NULL);
SetupOutputs(tv_worker);
if (TmThreadSpawn(tv_worker) != TM_ECODE_OK) {
printf("ERROR: TmThreadSpawn failed\n");
exit(EXIT_FAILURE);
}
}
return 0;
}
int main(int argc, char **argv)
{
/* limits */
int niter = 1;
int nscambi = 100000;
/* threads */
int cpu_white, cpu_black, cpu_main;
int white_rank = 0, black_rank = 61;
void * ch[2];
/* statistics */
struct timeval start;
struct timeval end;
/* 0 current, 1 sum, 2 square sum, 3 max value */
uint64_t result_test[4] = { 0, 0, 0, 0 };
double elapsed_test[4] = { 0, 0, 0, 0 };
double avg_Tscambio[4] = { 0 };
double sdev_Tscambio[4] = { 0 };
double max_Tscambio[4] = { 0 };
/* 0 results, 1 elapsed_test, 2 Tscambio, 3 avg_Tscambio, 4 sdev_Tscambio, 5 max_Tscambio */
double avg[6];
double sdev[6];
/* others */
cpu_set_t dp, udn_hardwall;
int i;
int retval[2];
int opt;
int longopt;
struct option options[] = {
{ "niter", required_argument, &longopt, 'n' },
{ "nscambi",required_argument, &longopt, 'm' },
{ "white", required_argument, &longopt, 'w' },
{ "black", required_argument, &longopt, 'b' },
{ NULL, 0, NULL, 0 }
};
while (longopt || -1 != (opt = getopt_long(argc, argv, "n:m:w:b:", options, NULL))) {
switch (opt) {
case 'n':
niter = atoi(optarg);
break;
case 'w':
white_rank = atoi(optarg);
break;
case 'b':
black_rank = atoi(optarg);
break;
case 'm':
nscambi = atoi(optarg);
break;
case 0:
opt=longopt;
continue;
}
longopt =0;
}
signal(SIGALRM, sighand_alrm);
/* defines cpus */
ERRHAND(tmc_cpus_get_dataplane_cpus(&dp));
if (tmc_cpus_count(&dp) < 3)
fprintf(stderr,
"[ERROR] numero di cpu dataplane disponibili non sufficiente\n");
//ERRHAND(cpu_white = tmc_cpus_find_first_cpu(&dp));
//ERRHAND(cpu_black = tmc_cpus_find_last_cpu(&dp));
ERRHAND(cpu_white = tmc_cpus_find_nth_cpu(&dp, white_rank));
ERRHAND(cpu_black = tmc_cpus_find_nth_cpu(&dp, black_rank));
ERRHAND(cpu_main = tmc_cpus_find_nth_cpu(&dp, tmc_cpus_count(&dp)-2));
/* bind this process to a dataplane cpu */
ERRHAND(tmc_cpus_set_my_cpu(cpu_main));
#if TEST_VERBOSE >= 1
printf("[INFO] main: cpu %d niter %d\n", tmc_cpus_get_my_cpu(), niter);
#endif
printf("main on cpu %d, white on cpu %d, black on cpu %d, "
"num of test iteration %d, num of exchanges %d\n",
tmc_cpus_get_my_cpu(), cpu_white, cpu_black, niter, nscambi);
/* define ansd initialize udn hardwall */
tmc_cpus_clear(&udn_hardwall);
ERRHAND(tmc_cpus_add_cpu(&udn_hardwall, cpu_main));
ERRHAND(tmc_cpus_add_cpu(&udn_hardwall, cpu_white));
ERRHAND(tmc_cpus_add_cpu(&udn_hardwall, cpu_black));
ERRHAND(tmc_udn_init(&dp));
/* init synchronization barriers */
ERRHAND_NZ(pthread_barrier_init(&computation_start, NULL, 2));
ERRHAND_NZ(pthread_barrier_init(&computation_end, NULL, 2));
for (i=0; i<niter; i++) {
arg_t arg[2];
Tscambio[1] = 0;
Tscambio[2] = 0;
Tscambio[3] = 0;
/* START TEST i-esimo */
ERRHAND(gettimeofday(&start, NULL));
/* set deadlock alarm */
alarm(deadlock_timeout);
/* setup environment */
ERRHAND_NN(ch[0] = ch_create(CH0_IMPL)(cpu_white, cpu_black CH0_CREATE_ARGS));
ERRHAND_NN(ch[1] = ch_create(CH1_IMPL)(cpu_black, cpu_white CH1_CREATE_ARGS));
arg[0].cpu = cpu_white;
arg[0].ch[0] = ch[0];
arg[0].ch[1] = ch[1];
arg[0].num_scambi = nscambi;
arg[1].cpu = cpu_black;
arg[1].ch[0] = ch[0];
arg[1].ch[1] = ch[1];
arg[1].num_scambi = nscambi;
/* start computation */
ERRHAND_NZ(pthread_create(&thread_white, NULL, task_pingpong_white,
(void *)&arg[0]));
ERRHAND_NZ(pthread_create(&thread_black, NULL, task_pingpong_black,
(void *)&arg[1]));
/* wait end of computation */
ERRHAND_NZ(pthread_join(thread_white, (void *)retval));
ERRHAND_NZ(pthread_join(thread_black, (void *)(retval+1)));
/* destroy environment */
ch_destroy(CH0_IMPL)(ch);
ch_destroy(CH1_IMPL)(ch+1);
/* END TEST i-esimo */
ERRHAND(gettimeofday(&end, NULL));
/* statistiche sugli scambi eseguiti nel test corrente */
calcStatistics(avg[2], sdev[2], Tscambio, nscambi);
timersub(&end, &start, &start);
prepareStatistics(elapsed_test, start.tv_sec*1000+start.tv_usec/(double)1000);
prepareStatistics(result_test, retval[0] + retval[1]);
prepareStatistics(avg_Tscambio, avg[2]);
prepareStatistics(sdev_Tscambio, sdev[2]);
prepareStatistics(max_Tscambio, Tscambio[3]);
#if TEST_VERBOSE == 0
//fprintf(stderr, "%d:%f:%f:%f:(%f);", i, avg[2]/2, sdev[2], (double)Tscambio[3]/2, avg_Tscambio[2]);
#elif TEST_VERBOSE >= 2
fprintf(stderr, printStatistics_format("Tscambio (cycles)", PRIu64)
"[STAT] Tsend (cycles):\n[STAT] %f\n",
printStatistics_values(avg[2], sdev[2], Tscambio),
avg[2]/(double)2
);
#endif /* TEST_VERBOSE == 0 */
deadlock_continue = 0;
} /* for (i=0; i<niter; i++) */
calcStatistics(avg[0], sdev[0], result_test, niter);
calcStatistics(avg[1], sdev[1], elapsed_test, niter);
calcStatistics(avg[3], sdev[3], avg_Tscambio, niter);
calcStatistics(avg[4], sdev[4], sdev_Tscambio, niter);
calcStatistics(avg[5], sdev[5], max_Tscambio, niter);
/*
fprintf(stderr,
printStatistics_format("Tscambio avg (cycles)", "f")
printStatistics_format("Tscambio sdev (cycles)", "f")
"[STAT] Tscambio max (cycles):\n[STAT] %f\n",
printStatistics_values(avg[3], sdev[3], avg_Tscambio),
printStatistics_values(avg[4], sdev[4], sdev_Tscambio),
maxmax_Tscambio
);
*/
/*
fprintf(stderr,
printStatistics_format2("Tscambio avg (cycles)", "f")
printStatistics_format2("Tscambio sdev ", "f")
printStatistics_format2("Tscambio max (cycles)", "f"),
printStatistics_values2(avg[3], sdev[3], avg_Tscambio),
printStatistics_values2(avg[4], sdev[4], sdev_Tscambio),
printStatistics_values2(avg[5], sdev[5], max_Tscambio)
);
Tscambio avg (cycles): 110.491957 0.258812 111.400840
Tscambio sdev : 118.790573 63.409627 306.372066
Tscambio max (cycles): 34756.240000 18675.977854 80419.000000
*/
fprintf(stderr,
"%-20s %-20s %-20s\n"
"%-20f %-20f %-20f\n",
"avg", "sdev", "max", avg[3], avg[4], avg[5]);
fprintf(stderr,
"\n\n"
" %-20s %-20s %-20s\n"
"Tscambio-avg: %-20f %-20f %-20f\n"
"Tscambio-dev: %-20f %-20f %-20f\n"
"Tscambio-max: %-20f %-20f %-20f\n",
"avg", "sdev", "max",
avg[3], avg[4], avg[5],
sdev[3], sdev[4], sdev[5],
avg_Tscambio[3], sdev_Tscambio[3], max_Tscambio[3]
);
#if TEST_VERBOSE == 0
#else
#endif /* TEST_VERBOSE == 0 */
return 0;
}
/**
* \brief RunModeTileMpipeWorkers set up to process all modules in each thread.
*
* \param iface pointer to the name of the interface from which we will
* fetch the packets
* \retval 0 if all goes well. (If any problem is detected the engine will
* exit())
*/
int RunModeTileMpipeWorkers(void)
{
SCEnter();
char tname[TM_THREAD_NAME_MAX];
char *thread_name;
TmModule *tm_module;
int pipe;
RunModeInitialize();
/* Available cpus */
uint16_t ncpus = UtilCpuGetNumProcessorsOnline();
TimeModeSetLive();
unsigned int pipe_max = 1;
if (ncpus > 1)
pipe_max = ncpus - 1;
intmax_t threads;
if (ConfGetInt("mpipe.threads", &threads) == 1) {
tile_num_pipelines = threads;
} else {
tile_num_pipelines = pipe_max;
}
SCLogInfo("%d Tilera worker threads", tile_num_pipelines);
ReceiveMpipeInit();
char *mpipe_dev = NULL;
int nlive = LiveGetDeviceCount();
if (nlive > 0) {
SCLogInfo("Using %d live device(s).", nlive);
/*mpipe_dev = LiveGetDevice(0);*/
} else {
/*
* Attempt to get interface from config file
* overrides -i from command line.
*/
if (ConfGet("mpipe.interface", &mpipe_dev) == 0) {
if (ConfGet("mpipe.single_mpipe_dev", &mpipe_dev) == 0) {
SCLogError(SC_ERR_RUNMODE, "Failed retrieving "
"mpipe.single_mpipe_dev from Conf");
exit(EXIT_FAILURE);
}
}
}
/* Get affinity for worker */
cpu_set_t cpus;
//int result = tmc_cpus_get_my_affinity(&cpus);
int result = tmc_cpus_get_dataplane_cpus(&cpus);
if (result < 0) {
SCLogError(SC_ERR_INVALID_ARGUMENT,
"tmc_cpus_get_my_affinity() returned=%d", result);
SCReturnInt(TM_ECODE_FAILED);
}
for (pipe = 0; pipe < tile_num_pipelines; pipe++) {
char *mpipe_devc;
if (nlive > 0) {
mpipe_devc = SCStrdup("multi");
} else {
mpipe_devc = SCStrdup(mpipe_dev);
}
if (unlikely(mpipe_devc == NULL)) {
printf("ERROR: SCStrdup failed for ReceiveMpipe\n");
exit(EXIT_FAILURE);
}
snprintf(tname, sizeof(tname), "%s#%02d", thread_name_workers, pipe+1);
/* create the threads */
ThreadVars *tv_worker =
TmThreadCreatePacketHandler(tname,
"packetpool", "packetpool",
"packetpool", "packetpool",
"pktacqloop");
if (tv_worker == NULL) {
printf("ERROR: TmThreadsCreate failed\n");
exit(EXIT_FAILURE);
}
tm_module = TmModuleGetByName("ReceiveMpipe");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName failed for ReceiveMpipe\n");
exit(EXIT_FAILURE);
}
TmSlotSetFuncAppend(tv_worker, tm_module, (void *)mpipe_devc);
/* Bind to a single cpu. */
int pipe_cpu = tmc_cpus_find_nth_cpu(&cpus, pipe);
tv_worker->rank = pipe;
TmThreadSetCPUAffinity(tv_worker, pipe_cpu);
tm_module = TmModuleGetByName("DecodeMpipe");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName DecodeMpipe failed\n");
exit(EXIT_FAILURE);
}
TmSlotSetFuncAppend(tv_worker, tm_module, NULL);
tm_module = TmModuleGetByName("StreamTcp");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName StreamTcp failed\n");
exit(EXIT_FAILURE);
}
TmSlotSetFuncAppend(tv_worker, tm_module, NULL);
if (DetectEngineEnabled()) {
tm_module = TmModuleGetByName("Detect");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName Detect failed\n");
exit(EXIT_FAILURE);
}
TmSlotSetFuncAppend(tv_worker, tm_module, NULL);
}
tm_module = TmModuleGetByName("RespondReject");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for RespondReject failed\n");
exit(EXIT_FAILURE);
}
TmSlotSetFuncAppend(tv_worker, tm_module, NULL);
SetupOutputs(tv_worker);
if (TmThreadSpawn(tv_worker) != TM_ECODE_OK) {
printf("ERROR: TmThreadSpawn failed\n");
exit(EXIT_FAILURE);
}
}
return 0;
}
int main(int argc, char **argv)
{
cpu_set_t dp;
int retval[2];
if (argc > 1)
niter = atoi(argv[1]);
signal(SIGALRM, sighand_alrm);
ERRHAND(tmc_cpus_get_dataplane_cpus(&dp));
ERRHAND(cpu_snd = tmc_cpus_find_first_cpu(&dp));
ERRHAND(cpu_rcv = tmc_cpus_find_last_cpu(&dp));
ERRHAND_NZ(pthread_barrier_init(&computation_end, NULL, 2));
for (cur_suite=0; cur_suite<niter; cur_suite++) {
PRINT(fprintf(stderr, "[INFO] start test suite %d\n", cur_suite));
/* --> init shared objects */
ERRHAND_NN(ch = ch_sym_ref_sm_create(M, cpu_snd, cpu_rcv, NULL));
/* set deadlock alarm */
alarm(deadlock_timeout);
/* start */
ERRHAND(gettimeofday(&start, NULL));
ERRHAND_NZ
(pthread_create(&thread_producer, NULL, task_producer, (void *)cpu_snd));
ERRHAND_NZ
(pthread_create(&thread_consumer, NULL, task_consumer, (void *)cpu_rcv));
/* wait end */
ERRHAND_NZ(pthread_join(thread_producer, (void *)retval));
ERRHAND_NZ(pthread_join(thread_consumer, (void *)(retval+1)));
/* end */
ERRHAND(gettimeofday(&end, NULL));
timersub(&end, &start, &start);
prepareStatistics(time_suite, start.tv_sec*1000+start.tv_usec/(double)1000);
prepareStatistics(result_suite, retval[0] + retval[1]);
PRINT(fprintf(stderr, "[INFO] end test suite %d: producer %d consumer %d\n",
cur_suite, retval[0], retval[1]));
/* --> destroy shared object */
ch_sym_ref_sm_destroy(&ch);
deadlock_continue = 0;
}
calcStatistics(avg[0], sdev[0], result_suite, niter);
calcStatistics(avg[1], sdev[1], time_suite, niter);
printf(printStatistics_format("return values", PRIu64)
printStatistics_format("elapsed time", "f")
printStatistics_format("Tcall-send", PRIu64)
printStatistics_format("Tcall-recv", PRIu64)
printStatistics_format("Tsend", PRIu64),
printStatistics_values(avg[0], sdev[0], result_suite),
printStatistics_values(avg[1], sdev[1], time_suite),
printStatistics_values(avg[2], sdev[2], call_send),
printStatistics_values(avg[3], sdev[3], call_recv),
printStatistics_values(avg[4], sdev[4], send)
);
return result_suite[1];
}
