static float performAveraging(void (*f)(void)){
initArray(arraySize, globalArray);
printArray(arraySize, globalArray);
uint64_t start, end;
uint64_t tsc_per_ms = 0;
sys_debug_get_tsc_per_ms(&tsc_per_ms);
printf("TSC PER MS = %" PRIu64 "\n", tsc_per_ms);
int repeats = 0;
float elapsed = 0;
if(loops < 1000000){repeats = 10;}
for(int i = 0; i<=repeats; i++){
start = rdtsc();
(*f)();
end = rdtsc();
uint64_t diff = (end - start) / tsc_per_ms;
float sample = (diff / 1000) + ((diff % 1000) / 1000.0);
elapsed += (1.0/(1+i)) * (sample - elapsed);
}
printArray(arraySize, globalArray);
return elapsed;
}
errval_t timing_sync_timer(void)
{
#if defined(__x86_64__) || defined(__i386__)
uint64_t tscperms;
errval_t err = sys_debug_get_tsc_per_ms(&tscperms);
assert(err_is_ok(err));
// Exponential backoff loop
for(uint64_t time_offset = MIN_DELAY_MS;
time_offset <= MAX_DELAY_MS;
time_offset *= 2) {
uint64_t synctime = rdtsc() + tscperms * time_offset;
int waitfor = 0;
received = 0;
error = SYS_ERR_OK;
for(int i = 0; i < MAX_CPUS; i++) {
struct intermon_binding *b = NULL;
err = intermon_binding_get(i, &b);
if(err_no(err) == MON_ERR_NO_MONITOR_FOR_CORE) {
continue;
}
assert(err_is_ok(err));
err = b->tx_vtbl.rsrc_timer_sync(b, NOP_CONT, synctime);
assert(err_is_ok(err));
waitfor++;
}
err = invoke_monitor_sync_timer(synctime);
if(err_is_fail(err)) {
error = err;
}
// Collect success/failure replies
while(received < waitfor) {
messages_wait_and_handle_next();
}
if(err_is_fail(error)) {
if(err_no(error) != SYS_ERR_SYNC_MISS) {
return error;
}
} else {
break;
}
}
return error;
#else
printf("Phase-locked local clocks not supported on this platform!\n");
return SYS_ERR_OK;
#endif
}
static int
rand_bench_time(char *target, uint8_t *buffer, size_t filesize, size_t blocksize, size_t count, size_t randval, struct bench_res *result)
{
size_t randbit;
size_t rsize = 0;
size_t wsize = 0;
int ret = 0;
#define RAND_NEXT do {\
randbit = ((randval >> 0) ^ (randval >> 3)) & 1;\
randval = (randval >> 1) | (randbit << (sizeof(size_t) * CHAR_BIT - 1));\
} while (0)
#define RAND_BLOCK ((randval % (filesize / blocksize)) * blocksize)
if (filesize % blocksize != 0) {
printf("Please spcifiy the filesize as a multiple of blocksize\n");
return 0;
}
errval_t err;
vfs_handle_t f = NULL;
char *path = vfs_path_mkabsolute(cwd, target);
err = vfs_open(path, &f);
if (err_is_fail(err)) {
printf("%s: %s\n", path, err_getstring(err));
return 1;
}
#if defined(__x86_64__) || defined(__i386__)
uint64_t tscperms;
err = sys_debug_get_tsc_per_ms(&tscperms);
assert(err_is_ok(err));
//printf("ticks per millisec: %" PRIu64 "\n", tscperms);
uint64_t start = rdtsc();
#endif
size_t count2 = count;
while (count2--) {
RAND_NEXT;
vfs_seek(f, VFS_SEEK_SET, RAND_BLOCK);
err = vfs_read(f, buffer, blocksize, &rsize);
if (err_is_fail(err)) {
DEBUG_ERR(err, "error reading file");
ret = 1;
goto out;
}
assert(rsize == blocksize);
}
#if defined(__x86_64__) || defined(__i386__)
uint64_t stop = rdtsc();
uint64_t elapsed_msecs = ((stop - start) / tscperms);
double elapsed_secs = (double)elapsed_msecs/1000.0;
result->read = elapsed_secs;
start = rdtsc();
#endif
count2 = count;
while(count2--) {
RAND_NEXT;
vfs_seek(f, VFS_SEEK_SET, RAND_BLOCK);
err = vfs_write(f, buffer, blocksize, &wsize);
if (err_is_fail(err) || wsize == 0) {
DEBUG_ERR(err, "error writing file");
ret = 1;
goto out;
}
assert(wsize == blocksize);
vfs_flush(f);
}
#if defined(__x86_64__) || defined(__i386__)
stop = rdtsc();
elapsed_msecs = ((stop - start) / tscperms);
elapsed_secs = (double)elapsed_msecs/1000.0;
result->write = elapsed_secs;
#endif
out:
if (f != NULL) {
err = vfs_close(f);
if (err_is_fail(err)) {
DEBUG_ERR(err, "in vfs_close");
}
}
return ret;
}
static double
dd_bench(char *source, char *target, size_t blocksize, size_t count)
{
vfs_handle_t source_vh = NULL;
vfs_handle_t target_vh = NULL;
size_t blocks_written = 0;
size_t total_bytes_read = 0;
size_t total_bytes_written = 0;
errval_t err;
double ret = 0;
double kbps = 0.0;
err = vfs_open(source, &source_vh);
if (err_is_fail(err)) {
printf("%s: %s (%ld)\n", source, err_getstring(err), err);
return -1;
}
err = vfs_create(target, &target_vh);
if (err_is_fail(err)) {
// close source handle
if (source_vh != NULL)
vfs_close(source_vh);
printf("%s: %s (%ld)\n", target, err_getstring(err), err);
return -1;
}
uint8_t *buffer = malloc(blocksize);
#if defined(__x86_64__) || defined(__i386__)
uint64_t tscperms;
err = sys_debug_get_tsc_per_ms(&tscperms);
assert(err_is_ok(err));
uint64_t start = rdtsc();
#endif
if (buffer == NULL) {
ret = -2;
printf("failed to allocate buffer of size %zd\n", blocksize);
goto out;
}
size_t rsize, wsize;
do {
err = vfs_read(source_vh, buffer, blocksize, &rsize);
if (err_is_fail(err)) {
DEBUG_ERR(err, "error reading file");
ret = -1;
goto out;
}
total_bytes_read += rsize;
size_t wpos = 0;
while (wpos < rsize) {
if (wpos > 0)
printf("was unable to write the whole chunk of size %zd. Now at pos: %zd of buffer\n", rsize, wpos);
err = vfs_write(target_vh, &buffer[wpos], rsize - wpos, &wsize);
if (err_is_fail(err) || wsize == 0) {
DEBUG_ERR(err, "error writing file");
ret = -1;
goto out;
}
wpos += wsize;
total_bytes_written += wsize;
}
blocks_written++;
} while (rsize > 0 && !(count > 0 && blocks_written >= count));
out:
if (buffer != NULL)
free(buffer);
if (source_vh != NULL) {
err = vfs_close(source_vh);
if (err_is_fail(err)) {
DEBUG_ERR(err, "in vfs_close");
}
}
if (target_vh != NULL) {
err = vfs_close(target_vh);
if (err_is_fail(err)) {
DEBUG_ERR(err, "in vfs_close");
}
}
#if defined(__x86_64__) || defined(__i386__)
uint64_t stop = rdtsc();
uint64_t elapsed_msecs = ((stop - start) / tscperms);
double elapsed_secs = (double)elapsed_msecs/1000.0;
kbps = ((double)total_bytes_written / 1024.0) / elapsed_secs;
if (ret == 0)
return kbps;
else
return ret;
#else
return ret;
#endif
}
static int
shuffle_file(int argc, char *argv[])
{
if (argc != 6) {
printf("Usage: %s <file> <filesize> <blocksize> <count> <seed>\n", argv[0]);
return 0;
}
size_t filesize = atoi(argv[2]);
size_t blocksize = atoi(argv[3]);
size_t count = atoi(argv[4]);
size_t randval = atoi(argv[5]);
size_t randbit;
char * filename = argv[1];
size_t rsize = 0;
size_t wsize = 0;
int ret = 0;
#define RAND_NEXT do {\
randbit = ((randval >> 0) ^ (randval >> 3)) & 1;\
randval = (randval >> 1) | (randbit << (sizeof(size_t) * CHAR_BIT - 1));\
} while (0)
#define RAND_BLOCK ((randval % (filesize / blocksize)) * blocksize)
if (filesize % blocksize != 0) {
printf("Please spcifiy the filesize as a multiple of blocksize\n");
return 0;
}
uint8_t * buffer = malloc(blocksize);
if (buffer == NULL) {
printf("failed to allocate buffer of size %zd\n", blocksize);
return 1;
}
errval_t err;
vfs_handle_t f = NULL;
char *path = vfs_path_mkabsolute(cwd, filename);
err = vfs_open(path, &f);
if (err_is_fail(err)) {
printf("%s: %s\n", path, err_getstring(err));
return 1;
}
#if defined(__x86_64__) || defined(__i386__)
uint64_t tscperms;
err = sys_debug_get_tsc_per_ms(&tscperms);
assert(err_is_ok(err));
//printf("ticks per millisec: %" PRIu64 "\n", tscperms);
uint64_t start = rdtsc();
#endif
size_t count2 = count;
while (count2--) {
RAND_NEXT;
vfs_seek(f, VFS_SEEK_SET, RAND_BLOCK);
err = vfs_read(f, buffer, blocksize, &rsize);
if (err_is_fail(err)) {
DEBUG_ERR(err, "error reading file");
ret = 1;
goto out;
}
assert(rsize == blocksize);
RAND_NEXT;
vfs_seek(f, VFS_SEEK_SET, RAND_BLOCK);
err = vfs_write(f, buffer, blocksize, &wsize);
if (err_is_fail(err) || wsize == 0) {
DEBUG_ERR(err, "error writing file");
ret = 1;
goto out;
}
assert(wsize == blocksize);
}
#if defined(__x86_64__) || defined(__i386__)
uint64_t stop = rdtsc();
uint64_t elapsed_msecs = ((stop - start) / tscperms);
double elapsed_secs = (double)elapsed_msecs/1000.0;
printf("start: %" PRIu64 " stop: %" PRIu64 "\n", start, stop);
double kbps = ((double)(count * blocksize) / 1024.0) / elapsed_secs;
printf("%zu bytes read. %zu bytes written. %f s, %f kB/s\n", count * blocksize, count * blocksize, elapsed_secs, kbps);
#endif
out:
if (buffer != NULL)
free(buffer);
if (f != NULL) {
err = vfs_close(f);
if (err_is_fail(err)) {
DEBUG_ERR(err, "in vfs_close");
}
}
return ret;
}
static errval_t
fill_bench(char *target, uint8_t *buffer, size_t blocksize, size_t count, struct bench_res *result)
{
vfs_handle_t target_vh = NULL;
size_t blocks_written = 0;
size_t blocks_read = 0;
size_t total_bytes_read = 0;
size_t total_bytes_written = 0;
uint64_t start = 0, stop = 0;
errval_t err;
#if defined(__x86_64__) || defined(__i386__)
uint64_t tscperms;
err = sys_debug_get_tsc_per_ms(&tscperms);
assert(err_is_ok(err));
#endif
errval_t ret = 0;
err = vfs_open(target, &target_vh);
if (err_is_fail(err)) {
printf("%s: %s (%ld)\n", target, err_getstring(err), err);
return err;
}
#if defined(__x86_64__) || defined(__i386__)
start = rdtsc();
#endif
if (buffer == NULL) {
ret = -2;
printf("failed to allocate buffer of size %zd\n", blocksize);
goto out;
}
size_t wsize;
do {
// initialize buffer
for (size_t i = 0; i < blocksize; i += sizeof(size_t))
*((size_t *)(buffer + i)) = blocks_written;
// write to file
size_t wpos = 0;
while (wpos < blocksize) {
if (wpos > 0)
printf("was unable to write the whole chunk of size %zd. Now at pos: %zd of buffer\n", blocksize, wpos);
err = vfs_write(target_vh, &buffer[wpos], blocksize - wpos, &wsize);
if (err_is_fail(err) || wsize == 0) {
DEBUG_ERR(err, "error writing file");
ret = err;
goto out;
}
wpos += wsize;
total_bytes_written += wsize;
}
blocks_written++;
} while (blocksize > 0 && !(count > 0 && blocks_written >= count));
err = vfs_close(target_vh);
if (err_is_fail(err)) {
DEBUG_ERR(err, "in vfs_close");
goto out;
}
#if defined(__x86_64__) || defined(__i386__)
stop = rdtsc();
{
uint64_t elapsed_msecs = ((stop - start) / tscperms);
double elapsed_secs = (double)elapsed_msecs/1000.0;
result->write = ((double)total_bytes_written / 1024.0) / elapsed_secs;
printf("%lf\n", result->write);
}
#endif
err = vfs_open(target, &target_vh);
if (err_is_fail(err)) {
printf("%s: %s (%ld)\n", target, err_getstring(err), err);
goto out;
}
err = vfs_seek(target_vh, VFS_SEEK_SET, 0);
if (err_is_fail(err)) {
DEBUG_ERR(err, "seeking failed");
ret = err;
goto out;
}
#if defined(__x86_64__) || defined(__i386__)
start = rdtsc();
#endif
size_t rsize;
do {
// read
size_t rpos = 0;
while (rpos < blocksize) {
if (rpos > 0)
printf("was unable to read whole chunk of size %zd. Now at pos: %zd of buffer\n", blocksize, rpos);
err = vfs_read(target_vh, &buffer[rpos], blocksize - rpos, &rsize);
if (err_is_fail(err) || wsize == 0) {
DEBUG_ERR(err, "error reading file");
ret = err;
goto out;
}
rpos += rsize;
total_bytes_read += rsize;
}
// verify data
for (size_t i = 0; i < blocksize; i += sizeof(size_t)) {
if (*((size_t *)(buffer + i)) != blocks_read) {
printf("Verification failed! Block %zd, value %zd\n", blocks_read, *((size_t *)(buffer + i)) );
ret = err;
goto out;
}
}
blocks_read++;
} while (blocksize > 0 && !(count > 0 && blocks_read >= count));
out:
if (target_vh != NULL) {
err = vfs_close(target_vh);
if (err_is_fail(err)) {
DEBUG_ERR(err, "in vfs_close");
ret = err;
}
}
#if defined(__x86_64__) || defined(__i386__)
stop = rdtsc();
{
uint64_t elapsed_msecs = ((stop - start) / tscperms);
double elapsed_secs = (double)elapsed_msecs/1000.0;
result->read = ((double)total_bytes_read / 1024.0) / elapsed_secs;
printf("%lf\n", result->read);
}
#endif
return ret;
}
static int dd(int argc, char *argv[])
{
// parse options
char *source = NULL;
char *target = NULL;
vfs_handle_t source_vh = NULL;
vfs_handle_t target_vh = NULL;
size_t blocksize = 512;
size_t count = 0;
size_t skip = 0;
size_t seek = 0;
size_t rsize = 0;
size_t wsize = 0;
size_t blocks_written = 0;
size_t total_bytes_read = 0;
size_t total_bytes_written = 0;
size_t progress = 0;
errval_t err;
int ret = 0;
for (int i = 1; i < argc; i++)
{
if (!strncmp(argv[i], "bs=", 3))
blocksize = atoi(argv[i] + 3);
else if (!strncmp(argv[i], "count=", 6))
count = atoi(argv[i] + 6);
else if (!strncmp(argv[i], "skip=", 5))
skip = atoi(argv[i] + 5);
else if (!strncmp(argv[i], "seek=", 5))
seek = atoi(argv[i] + 5);
else if (!strncmp(argv[i], "if=", 3))
source = (argv[i] + 3);
else if (!strncmp(argv[i], "of=", 3))
target = (argv[i] + 3);
else if (!strncmp(argv[i], "progress", 8))
progress = 1;
}
size_t one_per_cent = (blocksize * count) / 100;
printf("from: %s to: %s bs=%zd count=%zd seek=%zd skip=%zd\n", source, target, blocksize, count, seek, skip);
if (source != NULL)
{
char *path = vfs_path_mkabsolute(cwd, source);
err = vfs_open(path, &source_vh);
free(path);
if (err_is_fail(err)) {
printf("%s: %s\n", source, err_getstring(err));
return 1;
}
if (skip != 0)
{
// TODO: skip
}
}
if (target != NULL)
{
char *path = vfs_path_mkabsolute(cwd, target);
err = vfs_create(path, &target_vh);
free(path);
if (err_is_fail(err)) {
// close source handle
if (source_vh != NULL)
vfs_close(source_vh);
printf("%s: %s\n", target, err_getstring(err));
return 1;
}
if (seek != 0)
{
// TODO: seek
}
}
uint8_t * buffer = malloc(blocksize);
#if defined(__x86_64__) || defined(__i386__)
uint64_t tscperms;
err = sys_debug_get_tsc_per_ms(&tscperms);
assert(err_is_ok(err));
//printf("ticks per millisec: %" PRIu64 "\n", tscperms);
uint64_t start = rdtsc();
#endif
if (buffer == NULL)
{
ret = 2;
printf("failed to allocate buffer of size %zd\n", blocksize);
goto out;
}
do
{
//printf("copying block\n");
size_t read_bytes = 0;
do {
err = vfs_read(source_vh, buffer, blocksize, &rsize);
if (err_is_fail(err)) {
DEBUG_ERR(err, "error reading file");
ret = 1;
goto out;
}
total_bytes_read += rsize;
read_bytes += rsize;
size_t wpos = 0;
while (wpos < rsize) {
if (wpos > 0)
printf("was unable to write the whole chunk of size %zd. Now at pos: %zd of buffer\n", rsize, wpos);
err = vfs_write(target_vh, &buffer[wpos], rsize - wpos, &wsize);
if (err_is_fail(err) || wsize == 0) {
DEBUG_ERR(err, "error writing file");
ret = 1;
goto out;
}
wpos += wsize;
total_bytes_written += wsize;
}
} while(read_bytes < blocksize);
blocks_written++;
if (progress && one_per_cent && total_bytes_written % one_per_cent == 0) {
printf(".");
}
//printf("block successfully copied. read: %zd. blocks written: %zd\n", rsize, blocks_written);
} while (rsize > 0 && !(count > 0 && blocks_written >= count));
if (progress) printf("\n");
out:
if (buffer != NULL)
free(buffer);
if (source_vh != NULL) {
err = vfs_close(source_vh);
if (err_is_fail(err)) {
DEBUG_ERR(err, "in vfs_close");
}
}
if (target_vh != NULL) {
err = vfs_close(target_vh);
if (err_is_fail(err)) {
DEBUG_ERR(err, "in vfs_close");
}
}
#if defined(__x86_64__) || defined(__i386__)
uint64_t stop = rdtsc();
uint64_t elapsed_msecs = ((stop - start) / tscperms);
double elapsed_secs = (double)elapsed_msecs/1000.0;
printf("start: %" PRIu64 " stop: %" PRIu64 "\n", start, stop);
double kbps = ((double)total_bytes_written / 1024.0) / elapsed_secs;
printf("%zd bytes read. %zd bytes written. %f s, %f kB/s\n", total_bytes_read, total_bytes_written, elapsed_secs, kbps);
#else
printf("%zd bytes read. %zd bytes written.\n", total_bytes_read, total_bytes_written);
#endif
return ret;
}
void benchmark_init(void)
{
errval_t err;
// Getting CPU frequency
err = sys_debug_get_tsc_per_ms(&tscperms);
assert(err_is_ok(err));
printf("tcp benchmark: init started\n");
bool ret = lwip_init(cardname, qi);
if (!ret) {
USER_PANIC("lwip_init failed!");
return;
}
printf("tcp benchmark: lwip init done\n");
buf_count = buffer_count;
// Initialize benchmark control
bench_ctl = bench_ctl_init(BENCH_MODE_FIXEDRUNS, 1, total_runs);
bench_ctl_dry_runs(bench_ctl, dry_runs);
if (is_server) {
printf("elb_tcp: Starting benchmark server...\n");
// listen on port
if (use_udp) {
if (udp_server_bm_init(server_port) != 0) {
USER_PANIC("udp_server_bm_init failed");
return;
}
} else {
if (tcp_server_bm_init(server_port) != 0) {
USER_PANIC("tcp_server_bm_init failed");
return;
}
}
} else { // is client
printf("elb_tcp: Starting benchmark client...\n");
if (use_udp) {
data_to_send = prepare_udp_buffer(payload_size);
assert(data_to_send != NULL);
// memset(data_to_send, 1, payload_size);
// onnect on ip/port
if (udp_client_bm_init(server_ip_addr, server_port) != 0) {
USER_PANIC("udp_server_bm_init failed");
return;
}
} else {
data_to_send = malloc(MAX_PAYLOAD);
assert(data_to_send != NULL);
// FIXME: make it cache aligned
memset(data_to_send, 1, payload_size);
// onnect on ip/port
if (tcp_client_bm_init(server_ip_addr, server_port) != 0) {
USER_PANIC("tcp_server_bm_init failed");
return;
}
} // end else: is_client
} // end else: udp
} // end function: benchmark_init
int main(int argc, char *argv[])
{
#ifndef __linux__
if(argc < 5) {
printf("Usage: %s tracefile nslaves mountdir mount-URL\n", argv[0]);
exit(EXIT_FAILURE);
}
assert(err_is_ok(sys_debug_get_tsc_per_ms(&tscperms)));
errval_t err = vfs_mkdir(argv[3]);
assert(err_is_ok(err));
printf("----------------------------------- VFS MOUNT\n");
err = vfs_mount(argv[3], argv[4]);
if(err_is_fail(err)) {
DEBUG_ERR(err, "vfs_mount");
}
printf("----------------------------------- VFS MOUNT DONE\n");
assert(err_is_ok(err));
#else
if(argc < 3) {
printf("Usage: %s tracefile nslaves\n", argv[0]);
exit(EXIT_FAILURE);
}
#endif
memset(&SlState, 0, sizeof(SlState));
//SlState.waitset = get_default_waitset();
//struct waitset ws;
//waitset_init(&ws);
//SlState.waitset = &ws;
char *tracefile = argv[1];
SlState.num_slaves = atoi(argv[2]);
printf("tracefile=%s\n", tracefile);
printf("reading dependency graph...\n");
// Parse trace file into memory records
struct trace_list tlist = {.head = NULL, .tail = NULL};
parse_tracefile(tracefile, &tlist);
// Build task graph. roots are nodes without dependencies
#ifndef __linux__
msg("[MASTER] My cpu is: %d\n", disp_get_core_id());
#endif
memset(&TG, 0, sizeof(TG));
build_taskgraph(&tlist, &TG);
//print_all_tasks(&TG);
//print_taskgraph(&TG);
//printf("TG entries:%d completed:%d stack_size:%d\n", TG.pes_nr, TG.pes_completed, TG.stack_nr);
msg("[MASTER] INITIALIZING BUFFERS...\n");
trace_bufs_init(&TG);
msg("[MASTER] CONNECTING TO SLAVES...\n");
slaves_connect(&TG);
msg("[MASTER] STARTING WORK...\n");
uint64_t start_ticks = rdtsc();
for (;;) {
/* enqueue work to the slaves */
for (int sid=0; sid < SlState.num_slaves; sid++) {
struct slave *sl = SlState.slaves + sid;
// try to assign a pid entry to a slave, if it doesn't hove one
if (sl->pe == NULL) {
sl->pe = tg_pop(&TG);
if (sl->pe == NULL) {
continue; /* no more tasks in the stack */
}
dmsg("[MASTER] assigned pid:%d to sl:%d (stack_nr:%d completed:%d total:%d bytes:%zd)\n",
sl->pe->pid, sid, TG.stack_nr, TG.pes_completed, TG.pes_nr, sl->pe->tes_size);
sl->pe->sl = sl;
slave_push_work(sl);
}
master_process_reqs();
}
if (TG.pes_completed == TG.pes_nr)
break;
}
uint64_t work_ticks = rdtsc() - start_ticks;
slaves_finalize();
uint64_t total_ticks = rdtsc() - start_ticks;
printf("[MASTER] replay done> cache:%s slaves:%d ticks:%" PRIu64
" (%lf ms) [total: %lfms]\n",
vfs_cache_str, SlState.num_slaves, work_ticks,
(double)work_ticks /(double)tscperms,
(double)total_ticks/(double)tscperms);
slaves_print_stats();
return 0;
}
/*
* Entry point of Multi-core Insense runtime.
*/
int main(int argc, char* argv[]) {
PRINTFMC("Cache line size: %dB\n", cache_line_size());
PRINTFMC("Main thread: %u\n", (unsigned) pthread_self());
errval_t err;
coreid_t mycore = disp_get_core_id();
if (argc == 2) {
num_to_span = atoi(argv[1]);
if(num_to_span==0)
all_spanned = true;
debug_printf("Spanning onto %d cores\n", num_to_span);
for (int i = 1; i < num_to_span; i++) {
err = domain_new_dispatcher(mycore + i, span_cb, NULL);
if (err_is_fail(err)) {
DEBUG_ERR(err, "failed span %d", i);
}
}
} else {
debug_printf("ERROR: Must specify number of cores to span\n");
return EXIT_FAILURE;
}
posixcompat_pthread_set_placement_fn(rrPlacement);
while (!all_spanned) {
thread_yield();
}
my_mutex_init(&shared_heap_mutex);
#if HEAPS == HEAP_PRIVATE // Private heaps
// Initialize mutex
if (pthread_mutex_init(&thread_lock, NULL ) != 0) {
PRINTF("Mutex initialization failed.\n");
return -1;
}
#endif
mainThread = pthread_self(); // Note the ID of the main thread.
// Create a list for storing references to p-threads
threadList = listCreate();
// Create map used to store memory locations of small heaps (using Thread safe list)
SHList = listCreate();
// Create map used to store memory locations what is allocated using malloc
mallocList = listCreate();
// Start recording execution time
#if TIMING
// CPU time
uint64_t start, end;
uint64_t tsc_per_ms = 0;
sys_debug_get_tsc_per_ms(&tsc_per_ms);
start = rdtsc();
#endif
// Call primordial_main.
primordial_main(NULL );
// Join all p-threads
if (threadList != NULL ) {
listJoinThreads(threadList);
}
// Stop recording execution time
#if TIMING
end = rdtsc();
uint64_t diff = (end - start) / tsc_per_ms;
float elapsed = (diff / 1000) + ((diff % 1000) / 1000.0);
printf("CPU: %f seconds elapsed\n", elapsed);
#endif
// Destroy lists and free memory
listDestroy(threadList);
listDestroy(SHList);
listDestroy(mallocList);
#if HEAPS == HEAP_PRIVATE
pthread_mutex_destroy(&thread_lock); // Destroy mutex lock used with pthreads
#endif
return 0;
}
int RCCE_init(int *argc, char ***argv)
{
int ue;
void *nothing = NULL;
assert(*argc >= 3);
setup_routes(*argc, *argv);
// save pointer to executable name for later insertion into the argument list
char *executable_name = (*argv)[0];
RCCE_NP = atoi(*(++(*argv)));
RC_REFCLOCKGHZ = atof(*(++(*argv)));
if(RC_REFCLOCKGHZ == 0) {
printf("Barrelfish RCCE extension: Computing reference clock GHz automatically...\n");
uint64_t tscperms;
errval_t err = sys_debug_get_tsc_per_ms(&tscperms);
assert(err_is_ok(err));
RC_REFCLOCKGHZ = ((double)tscperms) / 1000000.0;
printf("Reference clock computed to be %.2g\n", RC_REFCLOCKGHZ);
}
// put the participating core ids (unsorted) into an array
for (ue=0; ue<RCCE_NP; ue++) {
RC_COREID[ue] = atoi(*(++(*argv)));
}
// make sure executable name is as expected
(*argv)[0] = executable_name;
RC_MY_COREID = MYCOREID();
// adjust apparent number of command line arguments, so it will appear to main
// program that number of UEs, clock frequency, and core ID list were not on
// command line
*argc -= RCCE_NP+2;
// sort array of participating phyical core IDs to determine their ranks
qsort((char *)RC_COREID, RCCE_NP, sizeof(int), id_compare);
// determine rank of calling core
for (ue=0; ue<RCCE_NP; ue++) {
if (RC_COREID[ue] == RC_MY_COREID) RCCE_IAM = ue;
}
// leave in one reassuring debug print
printf("My rank is %d, physical core ID is %d\n", RCCE_IAM, RC_MY_COREID);
if (RCCE_IAM<0) {
return(RCCE_ERROR_CORE_NOT_IN_HOSTFILE);
}
// create global communicator (equivalent of MPI_COMM_WORLD); this will also allocate
// the two synchronization flags associated with the global barrier
RCCE_comm_split(RCCE_global_color, nothing, &RCCE_COMM_WORLD);
#ifdef MEASURE_TIME
measure_start = RCCE_wtime();
measure_rcce_time = 0.0;
#endif
#ifdef MEASURE_DATA
memset(measure_rcce_data, 0, sizeof(measure_rcce_data));
#endif
return (RCCE_SUCCESS);
}
