/*
* queue add class_name qdisc_name [-size num_slots] [-weight value] [-delay delay_microsec]
* queue show
* queue del queue_number
* queue mod queue_number [-weight value] [-delay delay_microsec]
* queue stats [queue_number]
*/
void queueCmd()
{
char *next_tok;
char cname[MAX_DNAME_LEN], qdisc[MAX_DNAME_LEN];
// the following parameters are set to default values which are sometimes overwritten
int num_slots = 0; // means, set to default
double weight = 1.0, delay = 2.0;
if ((next_tok = strtok(NULL, " \n")) != NULL)
{
if (!strcmp(next_tok, "add"))
{
next_tok = strtok(NULL, " \n");
strcpy(cname, next_tok);
if (getClassDef(classifier, cname) == NULL)
{
verbose(1, "[queue]:: class name %s not defined ..", cname);
return;
}
next_tok = strtok(NULL, " \n");
strcpy(qdisc, next_tok);
if (lookupQDisc(pcore->qdiscs, qdisc) < 0)
{
verbose(1, "[queue]:: qdisc %s not defined .. ", qdisc);
return;
}
while ((next_tok = strtok(NULL, " \n")) != NULL)
{
if (!strcmp(next_tok, "-size"))
{
next_tok = strtok(NULL, " \n");
num_slots = atoi(next_tok);
}
else if (!strcmp(next_tok, "-weight"))
{
next_tok = strtok(NULL, " \n");
weight = atof(next_tok);
}
else if (!strcmp(next_tok, "-delay"))
{
next_tok = strtok(NULL, " \n");
delay = atof(next_tok);
}
}
addPktCoreQueue(pcore, cname, qdisc, weight, delay, num_slots);
}
else if (!strcmp(next_tok, "show"))
printAllQueues(pcore);
else if (!strcmp(next_tok, "del"))
{
if ((next_tok = strtok(NULL, " \n")) != NULL)
delPktCoreQueue(pcore, next_tok);
}
else if (!strcmp(next_tok, "mod"))
{
if ((next_tok = strtok(NULL, " \n")) != NULL)
{
strcpy(cname, next_tok);
if ((next_tok = strtok(NULL, " \n")) != NULL)
{
if (!strcmp(next_tok, "-weight"))
{
next_tok = strtok(NULL, " \n");
weight = atof(next_tok);
modifyQueueWeight(pcore, cname, weight);
}
else if (!strcmp(next_tok, "-qdisc"))
{
next_tok = strtok(NULL, " \n");
modifyQueueDiscipline(pcore, cname, next_tok);
}
}
}
}
else if (!strcmp(next_tok, "stats"))
printQueueStats(pcore);
}
}
void* __ctBackgroundThreadWriter(void* d)
{
FILE* serialFile;
char* fname = getenv("CONTECH_FE_FILE");
unsigned int wpos = 0;
unsigned int maxBuffersAlloc = 0, memLimitBufCount = 0;
size_t totalWritten = 0;
pct_serial_buffer memLimitQueue = NULL;
pct_serial_buffer memLimitQueueTail = NULL;
unsigned long long totalLimitTime = 0, startLimitTime, endLimitTime;
int mpiRank = __ctGetMPIRank();
int mpiPresent = __ctIsMPIPresent();
// TODO: Create MPI event
// TODO: Modify filename with MPI rank
// TODO: Only do the above when MPI is present
if (fname == NULL)
{
if (mpiPresent != 0)
{
char* fnameMPI = strdup("/tmp/contech_fe ");
fnameMPI[15] = '.';
snprintf(fnameMPI + 16, 5, "%d", mpiRank);
serialFile = fopen(fnameMPI, "wb");
free(fnameMPI);
}
else
{
serialFile = fopen("/tmp/contech_fe", "wb");
}
}
else
{
serialFile = fopen(fname, "wb");
}
if (serialFile == NULL)
{
fprintf(stderr, "Failure to open front-end stream for writing.\n");
if (fname == NULL) { fprintf(stderr, "\tCONTECH_FE_FILE unspecified\n");}
else {fprintf(stderr, "\tAttempted on %s\n", fname);}
exit(-1);
}
{
unsigned int id = 0;
ct_event_id ty = ct_event_version;
unsigned int version = CONTECH_EVENT_VERSION;
uint8_t* bb_info = _binary_contech_bin_start;
fwrite(&id, sizeof(unsigned int), 1, serialFile);
fwrite(&ty, sizeof(unsigned int), 1, serialFile);
fwrite(&version, sizeof(unsigned int), 1, serialFile);
fwrite(bb_info, sizeof(unsigned int), 1, serialFile);
totalWritten += 4 * sizeof(unsigned int);
{
size_t tl, wl;
unsigned int buf[2];
buf[0] = ct_event_rank;
buf[1] = mpiRank;
tl = 0;
do
{
wl = fwrite(&buf + tl, sizeof(unsigned int), 2 - tl, serialFile);
//if (wl > 0)
// wl is 0 on error, so it is safe to still add
tl += wl;
} while (tl < 2);
totalWritten += 2 * sizeof(unsigned int);
}
bb_info += 4; // skip the basic block count
while (bb_info != _binary_contech_bin_end)
{
// id, len, memop_0, ... memop_len-1
// Contech pass lays out the events in appropriate format
size_t tl = fwrite(bb_info, sizeof(char), _binary_contech_bin_end - bb_info, serialFile);
bb_info += tl;
totalWritten += tl;
}
}
// Main loop
// Write queued buffer to disk until program terminates
pthread_mutex_lock(&__ctQueueBufferLock);
do {
struct timespec ts;
clock_gettime(CLOCK_REALTIME, &ts);
ts.tv_sec += 30;
int condRetVal = 0;
// Check for queued buffer, i.e. is the program generating events
while (__ctQueuedBuffers == NULL && condRetVal == 0)
{
condRetVal = pthread_cond_timedwait(&__ctQueueSignal, &__ctQueueBufferLock, &ts);
}
if (condRetVal == 0)
{
pthread_mutex_unlock(&__ctQueueBufferLock);
}
// The thread writer will likely sit in this loop except when the memory limit is triggered
while (__ctQueuedBuffers != NULL)
{
// Write buffer to file
size_t tl = 0;
size_t wl = 0;
pct_serial_buffer qb = __ctQueuedBuffers;
// First craft the marker event that indicates a new buffer in the event list
// This event tells eventLib which contech created the next set of bytes
{
unsigned int buf[3];
buf[0] = ct_event_buffer;
buf[1] = __ctQueuedBuffers->id;
buf[2] = __ctQueuedBuffers->basePos;
//fprintf(stderr, "%d, %llx, %d\n", __ctQueuedBuffers->id, totalWritten, __ctQueuedBuffers->pos);
do
{
wl = fwrite(&buf + tl, sizeof(unsigned int), 3 - tl, serialFile);
//if (wl > 0)
// wl is 0 on error, so it is safe to still add
tl += wl;
} while (tl < 3);
totalWritten += 3 * sizeof(unsigned int);
}
// TODO: fully integrate into debug framework
#if DEBUG
if (totalWritten < 256)
{
int i;
for (i = 0; i < 256; i ++)
{
fprintf(stderr, "%x ", __ctQueuedBuffers->data[i]);
}
}
#endif
// Now write the bytes out of the buffer, until all have been written
tl = 0;
wl = 0;
if (qb->pos > SERIAL_BUFFER_SIZE)
{
fprintf(stderr, "Illegal buffer size - %d\n", qb->pos);
}
while (tl < __ctQueuedBuffers->pos)
{
wl = fwrite(__ctQueuedBuffers->data + tl,
sizeof(char),
(__ctQueuedBuffers->pos) - tl,
serialFile);
// if (wl < 0)
// {
// continue;
// }
tl += wl;
if (qb != __ctQueuedBuffers)
{
fprintf(stderr, "Tampering with __ctQueuedBuffers!\n");
}
}
if (tl != __ctQueuedBuffers->pos)
{
fprintf(stderr, "Write quantity(%lu) is not bytes in buffer(%d)\n", tl, __ctQueuedBuffers->pos);
}
totalWritten += tl;
// "Free" buffer
// First move the queue pointer, as we implicitly held the first element
// Then switch locks and put this processed buffer onto the free list
pthread_mutex_lock(&__ctQueueBufferLock);
{
pct_serial_buffer t = __ctQueuedBuffers;
__ctQueuedBuffers = __ctQueuedBuffers->next;
if (__ctQueuedBuffers == NULL) __ctQueuedBufferTail = NULL;
pthread_mutex_unlock(&__ctQueueBufferLock);
if (t->length < SERIAL_BUFFER_SIZE)
{
free(t);
// After the unlock is end of while loop,
// so as the buffer was free() rather than put on the list
// we continue
continue;
}
//continue; // HACK: LEAK!
// Switching locks, "t" is now only held locally
pthread_mutex_lock(&__ctFreeBufferLock);
#ifdef DEBUG
pthread_mutex_lock(&__ctPrintLock);
fprintf(stderr, "f,%p,%d\n", t, t->id);
fflush(stderr);
pthread_mutex_unlock(&__ctPrintLock);
#endif
if (__ctCurrentBuffers == __ctMaxBuffers)
{
if (__ctQueuedBuffers == NULL)
{
// memlimit end
struct timeb tp;
ftime(&tp);
endLimitTime = tp.time*1000 + tp.millitm;
totalLimitTime += (endLimitTime - startLimitTime);
memLimitQueueTail->next = t;
t->next = __ctFreeBuffers;
__ctFreeBuffers = memLimitQueue;
maxBuffersAlloc = __ctCurrentBuffers;
// N.B. It is possible that thread X is holding a lock L
// and then attempts to queue and allocate a new buffer.
// And that thread Y blocks on lock L, whereby its buffer
// will not be in the queue and therefore the count should
// be greater than 0.
__ctCurrentBuffers = __ctMaxBuffers - (memLimitBufCount + 1);
memLimitQueue = NULL;
memLimitQueueTail = NULL;
pthread_cond_broadcast(&__ctFreeSignal);
}
else
{
if (memLimitQueueTail == NULL)
{
memLimitBufCount = 1;
memLimitQueue = t;
memLimitQueueTail = t;
t->next = NULL;
// memlimit start
struct timeb tp;
ftime(&tp);
startLimitTime = tp.time*1000 + tp.millitm;
}
else
{
memLimitBufCount ++;
memLimitQueueTail->next = t;
t->next = NULL;
memLimitQueueTail = t;
}
}
}
else
{
t->next = __ctFreeBuffers;
__ctFreeBuffers = t;
if (__ctCurrentBuffers > maxBuffersAlloc)
{
maxBuffersAlloc = __ctCurrentBuffers;
}
assert(__ctCurrentBuffers > 0);
__ctCurrentBuffers --;
#if DEBUG
if (__ctCurrentBuffers < 2)
{
printf("%p\n", &t->data);
}
#endif
}
}
pthread_mutex_unlock(&__ctFreeBufferLock);
}
// Exit condition is # of threads exited = # of threads
// N.B. Main is part of this count
pthread_mutex_lock(&__ctQueueBufferLock);
if (__ctThreadExitNumber == __ctThreadGlobalNumber &&
__ctQueuedBuffers == NULL)
{
// destroy mutex, cond variable
// TODO: free freedBuffers
{
struct timeb tp;
ftime(&tp);
printf("CT_COMP: %d.%03d\n", (unsigned int)tp.time, tp.millitm);
printf("CT_LIMIT: %llu.%03llu\n", totalLimitTime / 1000, totalLimitTime % 1000);
}
printf("Total Contexts: %u\n", __ctThreadGlobalNumber);
printf("Total Uncomp Written: %ld\n", totalWritten);
printf("Max Buffers Alloc: %u of %lu\n", maxBuffersAlloc, sizeof(ct_serial_buffer_sized));
{
struct rusage use;
if (0 == getrusage(RUSAGE_SELF, &use))
{
printf("Max RSS: %ld\n", use.ru_maxrss);
}
}
printQueueStats();
fflush(stdout);
fflush(serialFile);
fclose(serialFile);
pthread_mutex_unlock(&__ctQueueBufferLock);
pthread_exit(NULL);
}
} while (1);
}
