void
ssmp_init_platf(int num_procs)
{
ssmp_num_ues_ = num_procs;
//initialize shared memory
tmc_cmem_init(0);
// Reserve the UDN rectangle that surrounds our cpus.
if (tmc_udn_init(&cpus) < 0)
tmc_task_die("Failure in 'tmc_udn_init(0)'.");
ssmp_barrier = (tmc_sync_barrier_t* ) tmc_cmem_calloc(SSMP_NUM_BARRIERS, sizeof (tmc_sync_barrier_t));
if (ssmp_barrier == NULL)
{
tmc_task_die("Failure in allocating mem for barriers");
}
uint32_t b;
for (b = 0; b < SSMP_NUM_BARRIERS; b++)
{
tmc_sync_barrier_init(ssmp_barrier + b, num_procs);
}
if (tmc_cpus_count(&cpus) < num_procs)
{
tmc_task_die("Insufficient cpus (%d < %d).", tmc_cpus_count(&cpus), num_procs);
}
tmc_task_watch_forked_children(1);
}
int
main(int argc, char**argv)
{
// Scan options.
char **args = &argv[1];
while (*args)
{
char *opt = NULL;
if ((opt = shift_option(&args, "--test_num=")))
g_run_multiplier = atoi(opt);
}
printf("This test will run %d times.\n",
((1 << LG2_CAPACITY) * g_run_multiplier));
// Make sure we have enough cpus.
// if (tmc_cpus_get_dataplane_cpus(&cpus) != 0)
if (tmc_cpus_get_my_affinity(&cpus) != 0)
tmc_task_die("tmc_cpus_get_my_affinity() failed.");
if (tmc_cpus_count(&cpus) < NUM_THREADS)
tmc_task_die("Insufficient cpus available.");
// Call the main thread function on each cpu, then wait for them all
// to exit.
run_threads(NUM_THREADS, thread_func);
finish_threads(NUM_THREADS);
return 0;
}
/*
* Clear counters for all tiles in cpu set
*/
int clear_all_counters(cpu_set_t *cpus) {
int num_of_cpus = tmc_cpus_count(cpus);
for (int i=0;i<num_of_cpus;i++) {
if (tmc_cpus_set_my_cpu(tmc_cpus_find_nth_cpu(cpus, i)) < 0) {
tmc_task_die("failure in 'tmc_set_my_cpu'");
return -1;
}
clear_counters();
}
return 0;
}
char *RunModeTileGetPipelineConfig(const char *custom_mode) {
intmax_t pipelines;
intmax_t detect_per_pipe;
intmax_t value = 0;
char *s;
if (custom_mode != NULL) {
return custom_mode;
}
char *runmode = NULL;
if (ConfGet("runmode", &runmode) == 1) {
if (strcmp(runmode, "workers") == 0) {
/* Available cpus */
cpu_set_t cpus;
tmc_cpus_get_dataplane_cpus(&cpus);
uint16_t ncpus = tmc_cpus_count(&cpus);
TileNumPipelines = ncpus - 1;
return runmode;
}
}
if (ConfGetInt("tile.pipelines", &pipelines) == 1) {
TileNumPipelines = pipelines;
} else {
TileNumPipelines = DFLT_TILERA_PIPELINES;
}
SCLogInfo("%d Tilera pipelines", TileNumPipelines);
if (ConfGetInt("tile.detect-per-pipeline", &detect_per_pipe) == 1) {
TileDetectThreadPerPipeline = detect_per_pipe;
} else {
TileDetectThreadPerPipeline = DFLT_DETECT_THREADS_PER_PIPELINE;
}
if ((ConfGetInt("mpipe.poll", &value)) == 1) {
/* only 1 and 2 are permitted */
if ((value >= 1) && (value <= 2)) {
TileNumPipelinesPerRx = (unsigned int) value;
} else {
SCLogError(SC_ERR_FATAL, "Illegal mpipe.poll value.");
}
}
if (ConfGet("tile.queue", &s) == 1) {
if (strcmp(s, "simple") == 0) {
queue_type = simple;
} else if (strcmp(s, "tmc") == 0) {
queue_type = tmc;
}
}
SCLogInfo("%d detect threads per pipeline", TileDetectThreadPerPipeline);
SCLogInfo("%d utilized dataplane tiles", (TILES_PER_PIPELINE * TileNumPipelines) + (TileNumPipelines / 2));
SCLogInfo("%s queueing between tiles", (queue_type == simple) ? "simple" : "tmc");
return NULL;
}
/*
* Setup counters for all tiles in cpu set
*/
int setup_all_counters(cpu_set_t *cpus) {
int num_of_cpus = tmc_cpus_count(cpus);
for (int i=0;i<num_of_cpus;i++) {
if (tmc_cpus_set_my_cpu(tmc_cpus_find_nth_cpu(cpus, i)) < 0) {
tmc_task_die("failure in 'tmc_set_my_cpu'");
return -1;
}
clear_counters();
setup_counters(LOCAL_WR_MISS, LOCAL_WR_CNT, LOCAL_DRD_MISS, LOCAL_DRD_CNT);
}
return 0;
}
int main(void) {
cpu_set_t cpus;
int wr_cnt, wr_miss, drd_cnt, drd_miss;
unsigned long start_cycles = get_cycle_count();
// Init cpus
if (tmc_cpus_get_my_affinity(&cpus) != 0) {
tmc_task_die("Failure in 'tmc_cpus_get_my_affinity()'.");
}
int num_cpus = tmc_cpus_count(&cpus);
printf("cpus_count is: %i\n", num_cpus);
// Setup Counters
setup_all_counters(&cpus);
unsigned long start_for = get_cycle_count();
unsigned long cycles[num_cpus];
unsigned int drd_cnts[num_cpus];
for (int i=0;i<num_cpus;i++) {
if (tmc_cpus_set_my_cpu(tmc_cpus_find_nth_cpu(&cpus, i)) < 0) {
tmc_task_die("failure in 'tmc_set_my_cpu'");
}
read_counters(&wr_cnt, &wr_miss, &drd_cnt, &drd_miss);
drd_cnts[i] = drd_cnt;
cycles[i] = get_cycle_count();
}
unsigned long end_for = get_cycle_count();
for (int i=1;i<num_cpus;i++) {
unsigned long temp = cycles[i] - cycles[i-1];
printf("time between %i and %i is %lu\n", i-1, i, temp);
printf("drd_cnt for tile %i was %i\n", i, drd_cnts[i]);
}
printf("Total cycles for-loop: %lu\n", end_for-start_for);
return 0;
}
/**
* \brief RunModeIdsTileMpipeWorkers set up the following thread packet handlers:
* - Receive thread (from iface pcap)
* - Decode thread
* - Stream thread
* - Detect: If we have only 1 cpu, it will setup one Detect thread
* If we have more than one, it will setup num_cpus - 1
* starting from the second cpu available.
* - Respond/Reject thread
* - Outputs thread
* By default the threads will use the first cpu available
* except the Detection threads if we have more than one cpu
*
* \param de_ctx pointer to the Detection Engine
* \param iface pointer to the name of the interface from which we will
* fetch the packets
* \retval 0 if all goes well. (If any problem is detected the engine will
* exit())
*/
int RunModeIdsTileMpipeWorkers(DetectEngineCtx *de_ctx) {
SCEnter();
char tname[32];
char *thread_name;
TmModule *tm_module;
int pipe;
char *detectmode = NULL;
int pool_detect_threads = 0;
extern TmEcode ReceiveMpipeInit(void); // move this
if (ConfGet("tile.detect", &detectmode) == 1) {
if (detectmode) {
SCLogInfo("DEBUG: detectmode %s", detectmode);
if (strcmp(detectmode, "pooled") == 0) {
pool_detect_threads = 1;
}
}
}
RunModeTileMpipeMapCores();
RunModeInitialize();
/* Available cpus */
cpu_set_t cpus;
tmc_cpus_get_dataplane_cpus(&cpus);
uint16_t ncpus = tmc_cpus_count(&cpus);
TimeModeSetLive();
int pipe_max = ncpus - 1;
TileNumPipelines = pipe_max;
TileNumPipelinesPerRx = 1;
ReceiveMpipeInit();
char *mpipe_dev = NULL;
int nlive = LiveGetDeviceCount();
if (nlive > 0) {
SCLogInfo("Using %d live device(s).", nlive);
/*mpipe_dev = LiveGetDevice(0);*/
} else {
/*
* Attempt to get interface from config file
* overrides -i from command line.
*/
if (ConfGet("mpipe.interface", &mpipe_dev) == 0) {
if (ConfGet("mpipe.single_mpipe_dev", &mpipe_dev) == 0) {
SCLogError(SC_ERR_RUNMODE, "Failed retrieving "
"mpipe.single_mpipe_dev from Conf");
exit(EXIT_FAILURE);
}
}
}
/*
* Careful. All of the pickup_queues must be created
* prior to building to pipeline so that the queues
* are adjacent in the lookup table. This lets the
* demux2 queue handler work.
*/
for (pipe = 0; pipe < pipe_max; pipe++) {
sprintf(pickup_queue[pipe], "pickup-queue%d", pipe);
if (TmqCreateQueue(pickup_queue[pipe]) == NULL) {
SCLogError(SC_ERR_RUNMODE, "Could not create pickup queue");
exit(EXIT_FAILURE);
}
}
for (pipe = 0; pipe < pipe_max; pipe++) {
char *mpipe_devc;
/* HACK: Receive Threads are shared between pairs of
* pipelines. So for every other pipeline create two
* queues and spawn only one thread.
*/
if (nlive > 0) {
mpipe_devc = SCStrdup("multi");
} else {
mpipe_devc = SCStrdup(mpipe_dev);
}
snprintf(tname, sizeof(tname), "Worker%d", pipe+1);
thread_name = SCStrdup(tname);
/* create the threads */
ThreadVars *tv_worker =
TmThreadCreatePacketHandler(thread_name,
"packetpool", "packetpool",
"packetpool", "packetpool",
"pktacqloop");
if (tv_worker == NULL) {
printf("ERROR: TmThreadsCreate failed\n");
exit(EXIT_FAILURE);
}
tm_module = TmModuleGetByName("ReceiveMpipe");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName failed for ReceiveMpipe\n");
exit(EXIT_FAILURE);
}
TmSlotSetFuncAppend(tv_worker, tm_module, (void *)mpipe_devc);
/* set affinity for worker */
TmThreadSetCPUAffinity(tv_worker, 1+pipe);
SCLogInfo("Thread %s pipe_max %d pipe %d cpu %d",
thread_name, pipe_max, pipe,
1+pipe);
tm_module = TmModuleGetByName("DecodeMpipe");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName DecodeMpipe failed\n");
exit(EXIT_FAILURE);
}
TmSlotSetFuncAppend(tv_worker,tm_module,NULL);
tm_module = TmModuleGetByName("StreamTcp");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName StreamTcp failed\n");
exit(EXIT_FAILURE);
}
TmSlotSetFuncAppend(tv_worker,tm_module,NULL);
tm_module = TmModuleGetByName("Detect");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName Detect failed\n");
exit(EXIT_FAILURE);
}
TmSlotSetFuncAppend(tv_worker,tm_module,(void *)de_ctx);
tm_module = TmModuleGetByName("RespondReject");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for RespondReject failed\n");
exit(EXIT_FAILURE);
}
TmSlotSetFuncAppend(tv_worker,tm_module,NULL);
SetupOutputs(tv_worker);
if (TmThreadSpawn(tv_worker) != TM_ECODE_OK) {
printf("ERROR: TmThreadSpawn failed\n");
exit(EXIT_FAILURE);
}
}
return 0;
}
int main(int argc, char **argv)
{
/* limits */
int niter = 1;
int nscambi = 100000;
/* threads */
int cpu_white, cpu_black, cpu_main;
int white_rank = 0, black_rank = 61;
void * ch[2];
/* statistics */
struct timeval start;
struct timeval end;
/* 0 current, 1 sum, 2 square sum, 3 max value */
uint64_t result_test[4] = { 0, 0, 0, 0 };
double elapsed_test[4] = { 0, 0, 0, 0 };
double avg_Tscambio[4] = { 0 };
double sdev_Tscambio[4] = { 0 };
double max_Tscambio[4] = { 0 };
/* 0 results, 1 elapsed_test, 2 Tscambio, 3 avg_Tscambio, 4 sdev_Tscambio, 5 max_Tscambio */
double avg[6];
double sdev[6];
/* others */
cpu_set_t dp, udn_hardwall;
int i;
int retval[2];
int opt;
int longopt;
struct option options[] = {
{ "niter", required_argument, &longopt, 'n' },
{ "nscambi",required_argument, &longopt, 'm' },
{ "white", required_argument, &longopt, 'w' },
{ "black", required_argument, &longopt, 'b' },
{ NULL, 0, NULL, 0 }
};
while (longopt || -1 != (opt = getopt_long(argc, argv, "n:m:w:b:", options, NULL))) {
switch (opt) {
case 'n':
niter = atoi(optarg);
break;
case 'w':
white_rank = atoi(optarg);
break;
case 'b':
black_rank = atoi(optarg);
break;
case 'm':
nscambi = atoi(optarg);
break;
case 0:
opt=longopt;
continue;
}
longopt =0;
}
signal(SIGALRM, sighand_alrm);
/* defines cpus */
ERRHAND(tmc_cpus_get_dataplane_cpus(&dp));
if (tmc_cpus_count(&dp) < 3)
fprintf(stderr,
"[ERROR] numero di cpu dataplane disponibili non sufficiente\n");
//ERRHAND(cpu_white = tmc_cpus_find_first_cpu(&dp));
//ERRHAND(cpu_black = tmc_cpus_find_last_cpu(&dp));
ERRHAND(cpu_white = tmc_cpus_find_nth_cpu(&dp, white_rank));
ERRHAND(cpu_black = tmc_cpus_find_nth_cpu(&dp, black_rank));
ERRHAND(cpu_main = tmc_cpus_find_nth_cpu(&dp, tmc_cpus_count(&dp)-2));
/* bind this process to a dataplane cpu */
ERRHAND(tmc_cpus_set_my_cpu(cpu_main));
#if TEST_VERBOSE >= 1
printf("[INFO] main: cpu %d niter %d\n", tmc_cpus_get_my_cpu(), niter);
#endif
printf("main on cpu %d, white on cpu %d, black on cpu %d, "
"num of test iteration %d, num of exchanges %d\n",
tmc_cpus_get_my_cpu(), cpu_white, cpu_black, niter, nscambi);
/* define ansd initialize udn hardwall */
tmc_cpus_clear(&udn_hardwall);
ERRHAND(tmc_cpus_add_cpu(&udn_hardwall, cpu_main));
ERRHAND(tmc_cpus_add_cpu(&udn_hardwall, cpu_white));
ERRHAND(tmc_cpus_add_cpu(&udn_hardwall, cpu_black));
ERRHAND(tmc_udn_init(&dp));
/* init synchronization barriers */
ERRHAND_NZ(pthread_barrier_init(&computation_start, NULL, 2));
ERRHAND_NZ(pthread_barrier_init(&computation_end, NULL, 2));
for (i=0; i<niter; i++) {
arg_t arg[2];
Tscambio[1] = 0;
Tscambio[2] = 0;
Tscambio[3] = 0;
/* START TEST i-esimo */
ERRHAND(gettimeofday(&start, NULL));
/* set deadlock alarm */
alarm(deadlock_timeout);
/* setup environment */
ERRHAND_NN(ch[0] = ch_create(CH0_IMPL)(cpu_white, cpu_black CH0_CREATE_ARGS));
ERRHAND_NN(ch[1] = ch_create(CH1_IMPL)(cpu_black, cpu_white CH1_CREATE_ARGS));
arg[0].cpu = cpu_white;
arg[0].ch[0] = ch[0];
arg[0].ch[1] = ch[1];
arg[0].num_scambi = nscambi;
arg[1].cpu = cpu_black;
arg[1].ch[0] = ch[0];
arg[1].ch[1] = ch[1];
arg[1].num_scambi = nscambi;
/* start computation */
ERRHAND_NZ(pthread_create(&thread_white, NULL, task_pingpong_white,
(void *)&arg[0]));
ERRHAND_NZ(pthread_create(&thread_black, NULL, task_pingpong_black,
(void *)&arg[1]));
/* wait end of computation */
ERRHAND_NZ(pthread_join(thread_white, (void *)retval));
ERRHAND_NZ(pthread_join(thread_black, (void *)(retval+1)));
/* destroy environment */
ch_destroy(CH0_IMPL)(ch);
ch_destroy(CH1_IMPL)(ch+1);
/* END TEST i-esimo */
ERRHAND(gettimeofday(&end, NULL));
/* statistiche sugli scambi eseguiti nel test corrente */
calcStatistics(avg[2], sdev[2], Tscambio, nscambi);
timersub(&end, &start, &start);
prepareStatistics(elapsed_test, start.tv_sec*1000+start.tv_usec/(double)1000);
prepareStatistics(result_test, retval[0] + retval[1]);
prepareStatistics(avg_Tscambio, avg[2]);
prepareStatistics(sdev_Tscambio, sdev[2]);
prepareStatistics(max_Tscambio, Tscambio[3]);
#if TEST_VERBOSE == 0
//fprintf(stderr, "%d:%f:%f:%f:(%f);", i, avg[2]/2, sdev[2], (double)Tscambio[3]/2, avg_Tscambio[2]);
#elif TEST_VERBOSE >= 2
fprintf(stderr, printStatistics_format("Tscambio (cycles)", PRIu64)
"[STAT] Tsend (cycles):\n[STAT] %f\n",
printStatistics_values(avg[2], sdev[2], Tscambio),
avg[2]/(double)2
);
#endif /* TEST_VERBOSE == 0 */
deadlock_continue = 0;
} /* for (i=0; i<niter; i++) */
calcStatistics(avg[0], sdev[0], result_test, niter);
calcStatistics(avg[1], sdev[1], elapsed_test, niter);
calcStatistics(avg[3], sdev[3], avg_Tscambio, niter);
calcStatistics(avg[4], sdev[4], sdev_Tscambio, niter);
calcStatistics(avg[5], sdev[5], max_Tscambio, niter);
/*
fprintf(stderr,
printStatistics_format("Tscambio avg (cycles)", "f")
printStatistics_format("Tscambio sdev (cycles)", "f")
"[STAT] Tscambio max (cycles):\n[STAT] %f\n",
printStatistics_values(avg[3], sdev[3], avg_Tscambio),
printStatistics_values(avg[4], sdev[4], sdev_Tscambio),
maxmax_Tscambio
);
*/
/*
fprintf(stderr,
printStatistics_format2("Tscambio avg (cycles)", "f")
printStatistics_format2("Tscambio sdev ", "f")
printStatistics_format2("Tscambio max (cycles)", "f"),
printStatistics_values2(avg[3], sdev[3], avg_Tscambio),
printStatistics_values2(avg[4], sdev[4], sdev_Tscambio),
printStatistics_values2(avg[5], sdev[5], max_Tscambio)
);
Tscambio avg (cycles): 110.491957 0.258812 111.400840
Tscambio sdev : 118.790573 63.409627 306.372066
Tscambio max (cycles): 34756.240000 18675.977854 80419.000000
*/
fprintf(stderr,
"%-20s %-20s %-20s\n"
"%-20f %-20f %-20f\n",
"avg", "sdev", "max", avg[3], avg[4], avg[5]);
fprintf(stderr,
"\n\n"
" %-20s %-20s %-20s\n"
"Tscambio-avg: %-20f %-20f %-20f\n"
"Tscambio-dev: %-20f %-20f %-20f\n"
"Tscambio-max: %-20f %-20f %-20f\n",
"avg", "sdev", "max",
avg[3], avg[4], avg[5],
sdev[3], sdev[4], sdev[5],
avg_Tscambio[3], sdev_Tscambio[3], max_Tscambio[3]
);
#if TEST_VERBOSE == 0
#else
#endif /* TEST_VERBOSE == 0 */
return 0;
}
