int SNetThreadingInit(int argc, char **argv)
{
#ifdef USE_CORE_AFFINITY
int i, num_cores;
#endif
char fname[32];
/* initialize the entity counter to 0 */
entity_count = 0;
(void) argc; /* NOT USED */
(void) argv; /* NOT USED */
#ifndef HAVE___THREAD
pthread_key_create(&thread_self_key, NULL);
#endif
#ifdef USE_CORE_AFFINITY
for (i=0; i<argc; i++) {
if(strcmp(argv[i], "-w") == 0 && i + 1 <= argc) {
/* Number of cores */
i = i + 1;
num_cores = atoi(argv[i]);
SetThreadAffinity( pthread_self(), MAXCORES, num_cores);
}
}
#endif
snprintf(fname, 31, "mon_n%02u_info.log", SNetDistribGetNodeId());
SNetThreadingMonitoringInit(fname);
return 0;
}
void SNetRouteDynamicEnter(snet_info_t *info, int dynamicIndex, int dynamicLoc,
snet_startup_fun_t fun)
{
snet_dest_t *dest = (snet_dest_t*) SNetInfoGetTag(info, prevDest);
dest->dynamicIndex = dynamicIndex;
int *counter = SNetMemAlloc(sizeof(int));
*counter = 0;
SNetInfoSetTag(info, infoCounter, (uintptr_t) counter, NULL);
if (fun != NULL) {
dest->parent = SNetNetToId(fun);
dest->parentNode = SNetDistribGetNodeId();
dest->dynamicLoc = dynamicLoc;
}
return;
}
int SNetThreadingInit(int argc, char **argv)
{
/* init map from file */
SNetLocvecMapInit(argc, argv);
#ifdef USE_LOGGING
char *mon_elts = NULL;
#endif
lpel_config_t config;
int i;
memset(&config, 0, sizeof(lpel_config_t));
config.type = DECEN_LPEL;
config.flags = LPEL_FLAG_PINNED; // pinned by default
/* task migration configure */
lpel_tm_config_t tm_conf;
memset(&tm_conf, 0, sizeof(lpel_tm_config_t));
int tm_mech = 0;
double tm_threshold = 0.0;
for (i=0; i<argc; i++) {
if(strcmp(argv[i], "-m") == 0 && i + 1 <= argc) {
/* Monitoring level */
i = i + 1;
#ifdef USE_LOGGING
mon_elts = argv[i];
#endif
} else if(strcmp(argv[i], "-excl") == 0 ) {
/* Assign realtime priority to workers*/
config.flags |= LPEL_FLAG_EXCLUSIVE;
} else if(strcmp(argv[i], "-dloc") == 0 ) {
/* Use distributed s-net location placement */
dloc_placement = true;
} else if(strcmp(argv[i], "-co") == 0 && i + 1 <= argc) {
/* Number of cores for others */
i = i + 1;
proc_others = atoi(argv[i]);
} else if(strcmp(argv[i], "-cw") == 0 && i + 1 <= argc) {
/* Number of cores for others */
i = i + 1;
proc_workers = atoi(argv[i]);
} else if(strcmp(argv[i], "-w") == 0 && i + 1 <= argc) {
/* Number of workers */
i = i + 1;
num_workers = atoi(argv[i]);
} else if(strcmp(argv[i], "-sosi") == 0) {
sosi_placement = true;
} else if (strcmp(argv[i], "-np") == 0) { // no pinned
config.flags ^= LPEL_FLAG_PINNED;
} else if (strcmp(argv[i], "-tm") == 0) {
i = i + 1;
tm_mech = atoi(argv[i]);
} else if (strcmp(argv[i], "-threshold") == 0) {
i = i +1;
tm_threshold = atof(argv[i]);
}
}
/* set up task migration configuration */
switch (tm_mech) {
case 1:
tm_conf.mechanism = LPEL_MIG_RAND;
if (tm_threshold <= 0)
tm_threshold = 0.5;
tm_conf.threshold = tm_threshold;
break;
case 2:
tm_conf.mechanism = LPEL_MIG_WAIT_PROP;
break;
default:
tm_conf.mechanism = LPEL_MIG_NONE;
break;
}
#ifdef USE_LOGGING
char fname[20+1];
if (mon_elts != NULL) {
if (strchr(mon_elts, MON_ALL_FLAG) != NULL) {
mon_flags = (1<<7) - 1;
} else {
if (strchr(mon_elts, MON_MAP_FLAG) != NULL) mon_flags |= SNET_MON_MAP;
if (strchr(mon_elts, MON_TIME_FLAG) != NULL) mon_flags |= SNET_MON_TIME;
if (strchr(mon_elts, MON_WORKER_FLAG) != NULL) mon_flags |= SNET_MON_WORKER;
if (strchr(mon_elts, MON_TASK_FLAG) != NULL) mon_flags |= SNET_MON_TASK;
if (strchr(mon_elts, MON_STREAM_FLAG) != NULL) mon_flags |= SNET_MON_STREAM;
if (strchr(mon_elts, MON_MESSAGE_FLAG) != NULL) mon_flags |= SNET_MON_MESSAGE;
if (strchr(mon_elts, MON_LOAD_FLAG) != NULL) mon_flags |= SNET_MON_LOAD;
}
/* set monitoring framework for task migration */
if (tm_conf.mechanism == LPEL_MIG_WAIT_PROP)
mon_flags |= SNET_MON_WAIT_PROP;
if ( mon_flags & SNET_MON_MAP) {
snprintf(fname, 20, "n%02d_tasks.map", SNetDistribGetNodeId() );
/* create a map file */
mapfile = fopen(fname, "w");
assert( mapfile != NULL);
(void) fprintf(mapfile, "%s%c", LOG_FORMAT_VERSION, END_LOG_ENTRY);
}
}
#endif
/* determine number of cpus */
if ( 0 != LpelGetNumCores( &num_cpus) ) {
SNetUtilDebugFatal("Could not determine number of cores!\n");
assert(0);
}
if (num_workers == -1)
config.num_workers = num_cpus;
else
config.num_workers = num_workers;
if (proc_workers == -1)
config.proc_workers = num_cpus;
else
config.proc_workers = proc_workers;
config.proc_others = proc_others;
#ifdef USE_LOGGING
/* initialise monitoring module */
config.mon.num_workers = config.num_workers;
SNetThreadingMonInit(&config.mon, SNetDistribGetNodeId(), mon_flags);
#endif
/* assin non-sosi task to non-sosi workers */
SNetAssignInit(config.num_workers);
LpelInit(&config);
/* init task migration */
tm_conf.num_workers = config.num_workers;
LpelTaskMigrationInit(&tm_conf);
/* start Lpel */
if (LpelStart(&config)) SNetUtilDebugFatal("Could not initialize LPEL!");
return 0;
}
int SNetThreadingInit(int argc, char **argv)
{
lpel_config_t config;
char fname[20+1];
int num_others = 0;
char *mon_elts = NULL;
LpelInit(&config);
config.flags = LPEL_FLAG_PINNED;
config.threshold = 0;
for (int i = 0; i < argc; i++) {
if(strcmp(argv[i], "-m") == 0 && i + 1 <= argc) {
/* Monitoring level */
i = i + 1;
mon_elts = argv[i];
} else if(strcmp(argv[i], "-excl") == 0 ) {
/* Assign realtime priority to workers*/
config.flags |= LPEL_FLAG_EXCLUSIVE;
} else if(strcmp(argv[i], "-dloc") == 0 ) {
/* Use distributed s-net location placement */
dloc_placement = true;
} else if(strcmp(argv[i], "-wo") == 0 && i + 1 <= argc) {
/* Number of cores for others */
i = i + 1;
num_others = atoi(argv[i]);
} else if(strcmp(argv[i], "-w") == 0 && i + 1 <= argc) {
/* Number of workers */
i = i + 1;
num_workers = atoi(argv[i]);
} else if(strcmp(argv[i], "-threshold") == 0 && i + 1 <= argc) {
/* Threshold for placement scheduler */
i = i + 1;
config.threshold = atof(argv[i]);
} else if(strcmp(argv[i], "-sockets") == 0 && i + 1 <= argc) {
i = i + 1;
config.sockets = atoi(argv[i]);
} else if(strcmp(argv[i], "-cores") == 0 && i + 1 <= argc) {
i = i + 1;
config.cores = atoi(argv[i]);
} else if(strcmp(argv[i], "-threads") == 0 && i + 1 <= argc) {
i = i + 1;
config.threads = atoi(argv[i]);
} else if(strcmp(argv[i], "-gather") == 0) {
config.gather = 1;
}
}
#ifdef USE_LOGGING
if (mon_elts != NULL) {
if (strchr(mon_elts, MON_ALL_FLAG) != NULL) {
mon_flags = (1<<7) - 1;
} else {
if (strchr(mon_elts, MON_MAP_FLAG) != NULL) mon_flags |= SNET_MON_MAP;
if (strchr(mon_elts, MON_TIME_FLAG) != NULL) mon_flags |= SNET_MON_TIME;
if (strchr(mon_elts, MON_WORKER_FLAG) != NULL) mon_flags |= SNET_MON_WORKER;
if (strchr(mon_elts, MON_TASK_FLAG) != NULL) mon_flags |= SNET_MON_TASK;
if (strchr(mon_elts, MON_STREAM_FLAG) != NULL) mon_flags |= SNET_MON_STREAM;
if (strchr(mon_elts, MON_MESSAGE_FLAG) != NULL) mon_flags |= SNET_MON_MESSAGE;
if (strchr(mon_elts, MON_LOAD_FLAG) != NULL) mon_flags |= SNET_MON_LOAD;
}
if ( mon_flags & SNET_MON_MAP) {
snprintf(fname, 20, "n%02d_tasks.map", SNetDistribGetNodeId() );
/* create a map file */
mapfile = fopen(fname, "w");
assert( mapfile != NULL);
(void) fprintf(mapfile, "%s%c", LOG_FORMAT_VERSION, END_LOG_ENTRY);
}
}
#endif
config.proc_others = num_others;
if (num_workers) {
config.proc_workers = num_workers;
config.num_workers = config.proc_others + config.proc_workers;
} else {
config.proc_workers = config.num_workers - config.proc_others;
num_workers = config.proc_workers;
}
#ifdef USE_LOGGING
/* initialise monitoring module */
SNetThreadingMonInit(&config.mon, SNetDistribGetNodeId(), mon_flags);
#endif
SNetAssignInit(config.num_workers);
if (LpelStart(&config)) SNetUtilDebugFatal("Could not initialize LPEL!");
return 0;
}
/* DripBack process a record. */
void SNetNodeDripBack(snet_stream_desc_t *desc, snet_record_t *rec)
{
landing_t *land = desc->landing;
dripback_arg_t *darg = LAND_NODE_SPEC(land, dripback);
trace(__func__);
if (land->type == LAND_dripback1) {
landing_dripback1_t *db1 = LAND_SPEC(land, dripback1);
landing_dripback2_t *db2 = LAND_SPEC(db1->dripback2, dripback2);
SNetFifoPut(db1->recfifo, rec);
if (FAA(&db2->queued, 1) == 0) {
if (db1->controldesc == NULL) {
SNetPushLanding(desc, db1->dripback2);
db1->controldesc = SNetStreamOpen(darg->selfref, desc);
}
rec = SNetRecCreate(REC_wakeup);
SNetWrite(&db1->controldesc, rec, true);
}
}
else if (land->type == LAND_dripback2) {
landing_dripback2_t *db2 = LAND_SPEC(land, dripback2);
bool via_access = (DESC_STREAM(desc) == darg->selfref);
assert(via_access || DESC_STREAM(desc) == darg->dripback);
while (rec) {
switch (REC_DESCR( rec)) {
case REC_data:
/* Test if record should go to the instance. */
if (SNetFeedbackMatch( rec, darg->back_patterns, darg->guards)) {
if (via_access) {
/* Because record came from outside add a detref counter. */
assert(db2->detfifo.head->next == NULL);
SNetRecDetrefAdd(rec, ++(db2->entered), land, &db2->detfifo);
} else {
/* Record came from the instance. */
assert(DATA_REC(rec, detref));
}
if (db2->instdesc == NULL) {
/* Instance should come back to this landing. */
if (SNetTopLanding(desc) != land) {
SNetPushLanding(desc, land);
}
db2->instdesc = SNetStreamOpen(darg->instance, desc);
if (SNetTopLanding(desc) == land) {
SNetPopLanding(desc);
SNetLandingDone(land);
}
}
SNetWrite(&db2->instdesc, rec, false);
} else {
if (!via_access) {
/* Record leaves the dripback loop. */
assert(DATA_REC(rec, detref));
SNetFeedbackLeave(rec, land, &db2->detfifo);
}
if (db2->outdesc == NULL) {
db2->outdesc = SNetStreamOpen(darg->output, desc);
}
SNetWrite(&db2->outdesc, rec, false);
}
break;
case REC_wakeup:
assert(via_access);
SNetRecDestroy(rec);
break;
case REC_terminate:
assert(via_access);
assert(db2->terminate == DripBackInitial);
db2->terminate = DripBackDraining;
SNetRecDestroy(rec);
break;
case REC_detref:
if (DETREF_REC( rec, leave) == land &&
DETREF_REC( rec, location) == SNetDistribGetNodeId())
{
assert(!via_access);
SNetDetLeaveCheckDetref(rec, &db2->detfifo);
if (DETREF_REC( rec, detref) == SNetFifoPeekFirst(&db2->detfifo)) {
DripBackCheckBusy(db2);
}
SNetRecDestroy(rec);
} else {
assert(via_access);
if (db2->outdesc == NULL) {
db2->outdesc = SNetStreamOpen(darg->output, desc);
}
SNetWrite(&db2->outdesc, rec, false);
}
break;
case REC_sync:
SNetRecDestroy(rec);
break;
default:
SNetRecUnknownEnt(__func__, rec, darg->entity);
}
rec = NULL;
if (db2->state == DripBackBusy) {
if (DripBackCheckBusy(db2) == false) {
assert(db2->queued > 0);
if (SAF(&db2->queued, 1) == 0) {
db2->state = DripBackIdle;
} else {
rec = SNetFifoGet(&db2->recfifo);
assert(rec);
via_access = true;
}
}
} else {
assert(db2->state == DripBackIdle);
assert(SNetFifoPeekFirst(&db2->detfifo) == NULL);
if (db2->queued > 0) {
rec = SNetFifoGet(&db2->recfifo);
assert(rec);
via_access = true;
db2->state = DripBackBusy;
}
}
}
if (db2->terminate == DripBackDraining) {
if (db2->state == DripBackIdle) {
assert(db2->queued == 0 && DripBackCheckBusy(db2) == false);
db2->terminate = DripBackTerminated;
}
}
if (db2->terminate == DripBackTerminated) {
if (db2->instdesc) {
snet_stream_desc_t *desc = db2->instdesc;
db2->instdesc = NULL;
SNetDescDone(desc);
SNetLandingDone(land);
}
}
}
else {
assert(0);
}
}
/*****************************************************************************
* Helper functions
****************************************************************************/
static void GenerateRecId(snet_record_id_t *rid)
{
assert( SNET_REC_SUBID_NUM == 2 );
rid->subid[0] = SNetAtomicCntFetchAndInc(&recid_sequencer);
rid->subid[1] = SNetDistribGetNodeId();
}
void startMonMPI() {
int node = SNetDistribGetNodeId();
char fn[1000];
sprintf(fn, "n%02d_comm.log", node);
mon_mpi = fopen(fn, "w");
}
