int main(int argc,
char **argv)
{
errval_t err;
debug_printf("Xeon Phi Test started on the card.\n");
err = xeon_phi_client_init(disp_xeon_phi_id());
EXPECT_SUCCESS(err, "xeon_phi_client_init");
xeon_phi_client_set_callbacks(&callbacks);
alloc_local();
wait_for_connection();
char iface[30];
snprintf(iface, 30, "xphi_ump_bench.%u", XPHI_BENCH_CORE_HOST);
debug_printf("sending open to host domain..\n");
err = xeon_phi_client_chan_open(disp_xeon_phi_id(), domainid, 0, local_frame, 2);
EXPECT_SUCCESS(err, "xeon_phi_client_init");
#if XPHI_BENCH_INITIATOR_HOST
debug_printf("giving time for host to initialize...\n");
for (uint32_t i = 0; i < 10; ++i) {
delay_ms(4000);
thread_yield();
}
#endif
#if XPHI_BENCH_INITIATOR_HOST
debug_printf("---------------- normal run -----------------\n");
xphi_bench_start_echo(&xphi_uc);
debug_printf("---------------- reversed run -----------------\n");
xphi_bench_start_echo(&xphi_uc_rev);
#else
#ifndef XPHI_BENCH_THROUGHPUT
debug_printf("---------------- normal run -----------------\n");
xphi_bench_start_initator_rtt(&xphi_uc);
debug_printf("---------------- reversed run -----------------\n");
xphi_bench_start_initator_rtt(&xphi_uc_rev);
#else
#ifdef XPHI_BENCH_SEND_SYNC
debug_printf("---------------- normal run -----------------\n");
xphi_bench_start_initator_sync(&xphi_uc);
debug_printf("---------------- reversed run -----------------\n");
xphi_bench_start_initator_sync(&xphi_uc_rev);
#else
debug_printf("---------------- normal run -----------------\n");
xphi_bench_start_initator_async(&xphi_uc);
debug_printf("---------------- reversed run -----------------\n");
xphi_bench_start_initator_async(&xphi_uc_rev);
#endif
#endif
#endif
}
static void init_buffer(void)
{
#ifdef __k1om__
#if XPHI_BENCH_CHAN_SEPARATED
if (disp_xeon_phi_id()) {
debug_printf("buffer configuration: in=remote_buf; out=local_buf\n");
debug_printf("buffer configuration (reversed): in=local_buf; out=remote_buf\n");
inbuf = remote_buf;
outbuf = local_buf;
inbuf_rev = local_buf + XPHI_BENCH_MSG_CHAN_SIZE;
outbuf_rev = remote_buf + XPHI_BENCH_MSG_CHAN_SIZE;
} else {
debug_printf("buffer configuration: in=remote_buf; out=local_buf\n");
debug_printf("buffer configuration (reversed): in=local_buf; out=remote_buf\n");
inbuf = remote_buf;
outbuf = local_buf;
inbuf_rev = local_buf + XPHI_BENCH_MSG_CHAN_SIZE;
outbuf_rev = remote_buf + XPHI_BENCH_MSG_CHAN_SIZE;
}
#else
if (disp_xeon_phi_id()) {
debug_printf("buffer configuration: in=local; out=local\n");
debug_printf("buffer configuration (reversed): in=remote; out=remote\n");
inbuf_rev = remote_buf;
outbuf_rev = remote_buf + XPHI_BENCH_MSG_CHAN_SIZE;
inbuf = local_buf;
outbuf = local_buf + XPHI_BENCH_MSG_CHAN_SIZE;
} else {
debug_printf("buffer configuration: in=remote; out=remote\n");
debug_printf("buffer configuration (reversed): in=local; out=local\n");
inbuf = remote_buf + XPHI_BENCH_MSG_CHAN_SIZE;
outbuf = remote_buf;
inbuf_rev = local_buf + XPHI_BENCH_MSG_CHAN_SIZE;
outbuf_rev = local_buf;
}
#endif
#else
#if XPHI_BENCH_CHAN_SEPARATED
debug_printf("buffer configuration: in=remote; out=local\n");
debug_printf("buffer configuration (reversed): in=local; out=remote\n");
inbuf = remote_buf;
outbuf = local_buf;
inbuf_rev = local_buf + XPHI_BENCH_MSG_CHAN_SIZE;
outbuf_rev = remote_buf + XPHI_BENCH_MSG_CHAN_SIZE;
#else
debug_printf("buffer configuration: in=remote; out=remote\n");
debug_printf("buffer configuration (reversed): in=local; out=local\n");
inbuf = remote_buf + XPHI_BENCH_MSG_CHAN_SIZE;
outbuf = remote_buf;
inbuf_rev = local_buf + XPHI_BENCH_MSG_CHAN_SIZE;
outbuf_rev = local_buf;
#endif
#endif
}
static errval_t alloc_local(void)
{
errval_t err;
size_t frame_size = 0;
if (disp_xeon_phi_id() == 0) {
frame_size = XPHI_BENCH_FRAME_SIZE_HOST;
} else {
frame_size = XPHI_BENCH_FRAME_SIZE_CARD;
}
if (!frame_size) {
frame_size = 4096;
}
debug_printf("Allocating a frame of size: %lx\n", frame_size);
size_t alloced_size = 0;
err = frame_alloc(&local_frame, frame_size, &alloced_size);
assert(err_is_ok(err));
assert(alloced_size >= frame_size);
struct frame_identity id;
err = invoke_frame_identify(local_frame, &id);
assert(err_is_ok(err));
local_base = id.base;
local_frame_sz = alloced_size;
err = vspace_map_one_frame(&local_buf, alloced_size, local_frame, NULL, NULL);
return err;
}
/**
* \brief Setup the dispatcher frame
*/
static errval_t spawn_setup_dispatcher(struct spawninfo *si,
coreid_t core_id,
const char *name,
genvaddr_t entry,
void* arch_info)
{
errval_t err;
/* Create dispatcher frame (in taskcn) */
si->dispframe.cnode = si->taskcn;
si->dispframe.slot = TASKCN_SLOT_DISPFRAME;
struct capref spawn_dispframe = {
.cnode = si->taskcn,
.slot = TASKCN_SLOT_DISPFRAME2,
};
err = frame_create(si->dispframe, (1 << DISPATCHER_FRAME_BITS), NULL);
if (err_is_fail(err)) {
return err_push(err, SPAWN_ERR_CREATE_DISPATCHER_FRAME);
}
err = cap_copy(spawn_dispframe, si->dispframe);
if (err_is_fail(err)) {
return err_push(err, SPAWN_ERR_CREATE_DISPATCHER_FRAME);
}
/* Map in dispatcher frame */
dispatcher_handle_t handle;
err = vspace_map_one_frame((void**)&handle, 1ul << DISPATCHER_FRAME_BITS,
si->dispframe, NULL, NULL);
if (err_is_fail(err)) {
return err_push(err, SPAWN_ERR_MAP_DISPATCHER_TO_SELF);
}
genvaddr_t spawn_dispatcher_base;
err = spawn_vspace_map_one_frame(si, &spawn_dispatcher_base, spawn_dispframe,
1UL << DISPATCHER_FRAME_BITS);
if (err_is_fail(err)) {
return err_push(err, SPAWN_ERR_MAP_DISPATCHER_TO_NEW);
}
/* Set initial state */
// XXX: Confusion address translation about l/gen/addr in entry
struct dispatcher_shared_generic *disp =
get_dispatcher_shared_generic(handle);
struct dispatcher_generic *disp_gen = get_dispatcher_generic(handle);
arch_registers_state_t *enabled_area =
dispatcher_get_enabled_save_area(handle);
arch_registers_state_t *disabled_area =
dispatcher_get_disabled_save_area(handle);
/* Place core_id */
disp_gen->core_id = core_id;
/* place eh information */
disp_gen->eh_frame = si->eh_frame;
disp_gen->eh_frame_size = si->eh_frame_size;
disp_gen->eh_frame_hdr = si->eh_frame_hdr;
disp_gen->eh_frame_hdr_size = si->eh_frame_hdr_size;
/* Setup dispatcher and make it runnable */
disp->udisp = spawn_dispatcher_base;
disp->disabled = 1;
disp->fpu_trap = 1;
#ifdef __k1om__
disp->xeon_phi_id = disp_xeon_phi_id();
#endif
// Copy the name for debugging
const char *copy_name = strrchr(name, '/');
if (copy_name == NULL) {
copy_name = name;
} else {
copy_name++;
}
strncpy(disp->name, copy_name, DISP_NAME_LEN);
spawn_arch_set_registers(arch_info, handle, enabled_area, disabled_area);
registers_set_entry(disabled_area, entry);
si->handle = handle;
return SYS_ERR_OK;
}
errval_t spawn_map_bootinfo(struct spawninfo *si, genvaddr_t *retvaddr)
{
errval_t err;
struct capref src = {
.cnode = cnode_task,
.slot = TASKCN_SLOT_BOOTINFO
};
struct capref dest = {
.cnode = si->taskcn,
.slot = TASKCN_SLOT_BOOTINFO
};
err = cap_copy(dest, src);
if (err_is_fail(err)) {
return err_push(err, LIB_ERR_CAP_COPY);
}
err = spawn_vspace_map_one_frame(si, retvaddr, dest, BOOTINFO_SIZE);
if (err_is_fail(err)) {
return err_push(err, SPAWN_ERR_MAP_BOOTINFO);
}
return SYS_ERR_OK;
}
/**
* \brief Retrive the commandline args of #name
*
* The arguments are malloced into a new space so need to be freed after use
*/
errval_t spawn_get_cmdline_args(struct mem_region *module,
char **retargs)
{
assert(module != NULL && retargs != NULL);
/* Get the cmdline args */
const char *args = getopt_module(module);
/* Allocate space */
*retargs = malloc(sizeof(char) * strlen(args));
if (!retargs) {
return LIB_ERR_MALLOC_FAIL;
}
/* Copy args */
strcpy(*retargs, args);
return SYS_ERR_OK;
}
int main(int argc,
char *argv[])
{
debug_printf("Xeon Phi module started on node [%u].\n", disp_xeon_phi_id());
errval_t err;
mmio_cap.cnode = cnode_task;
sysmem_cap.cnode = cnode_task;
assert(!capref_is_null(mmio_cap));
assert(!capref_is_null(sysmem_cap));
xphi.is_client = 0x1;
xphi.id = disp_xeon_phi_id();
for (uint32_t i = 0; i < XEON_PHI_NUM_MAX; ++i) {
xphi.topology[i].id = i;
xphi.topology[i].local = &xphi;
}
XDEBUG("Initializing system memory cap manager...\n");
err = sysmem_cap_manager_init(sysmem_cap);
if (err_is_fail(err)) {
USER_PANIC_ERR(err, "Could not initialize the cap manager.\n");
}
err = map_mmio_space(&xphi);
if (err_is_fail(err)) {
USER_PANIC_ERR(err, "could not map the mmio space");
}
err = xdma_service_init(&xphi);
if (err_is_fail(err)) {
USER_PANIC_ERR(err, "Could not initialize the dma engine.\n");
}
err = smpt_init(&xphi);
if (err_is_fail(err)) {
USER_PANIC_ERR(err, "Could not initialize the SMTP.\n");
}
/* wait until the kernels are booted and spawnds are ready */
err = nameservice_blocking_lookup("all_spawnds_up", NULL);
if (err_is_fail(err)) {
USER_PANIC_ERR(err, "all_spawnds_up.\n");
}
//dma_impl_test(&xphi);
lpaddr_t host_msg_base = strtol(argv[0], NULL, 16);
uint8_t host_msg_size = strtol(argv[1], NULL, 16);
XMESSAGING_DEBUG("Getting the host messaging cap...[%016lx, %02x]\n",
host_msg_base, host_msg_size);
err = sysmem_cap_request(host_msg_base, host_msg_size, &host_cap);
if (err_is_fail(err)) {
USER_PANIC_ERR(err, "Could not obtain the system messsaging cap\n");
}
err = interphi_init(&xphi, host_cap);
if (err_is_fail(err)) {
USER_PANIC_ERR(err, "Could not initialize the interphi communication\n");
}
err = xeon_phi_service_init(&xphi);
if (err_is_fail(err)) {
USER_PANIC_ERR(err, "could not initialize the messaging service");
}
XMESSAGING_DEBUG("Start polling for messages...\n");
while (1) {
uint8_t idle = 0x1;
err = xdma_service_poll(&xphi);
idle = idle && (err_no(err) == DMA_ERR_DEVICE_IDLE);
err = event_dispatch_non_block(get_default_waitset());
if (err_is_fail(err)) {
if ((err_no(err) == LIB_ERR_NO_EVENT) && idle) {
thread_yield();
continue;
}
if (err_no(err) != LIB_ERR_NO_EVENT) {
USER_PANIC_ERR(err, "msg loop");
}
}
}
XDEBUG("Messaging loop terminated...\n");
return 0;
}
/**
* \brief initializes the XOMP worker library
*
* \param wid Xomp worker id
*
* \returns SYS_ERR_OK on success
* errval on failure
*/
errval_t xomp_worker_init(xomp_wid_t wid)
{
errval_t err;
worker_id = wid;
XWI_DEBUG("initializing worker {%016lx} iref:%u\n", worker_id, svc_iref);
#if XOMP_BENCH_WORKER_EN
bench_init();
#endif
struct capref frame = {
.cnode = cnode_root,
.slot = ROOTCN_SLOT_ARGCN
};
struct frame_identity id;
err = invoke_frame_identify(frame, &id);
if (err_is_fail(err)) {
return err_push(err, XOMP_ERR_INVALID_MSG_FRAME);
}
size_t frame_size = 0;
if (svc_iref) {
frame_size = XOMP_TLS_SIZE;
} else {
frame_size = XOMP_FRAME_SIZE;
err = spawn_symval_cache_init(0);
if (err_is_fail(err)) {
return err;
}
}
if ((1UL << id.bits) < XOMP_TLS_SIZE) {
return XOMP_ERR_INVALID_MSG_FRAME;
}
msgframe = frame;
err = vspace_map_one_frame(&msgbuf, frame_size, frame, NULL, NULL);
if (err_is_fail(err)) {
err_push(err, XOMP_ERR_WORKER_INIT_FAILED);
}
if (svc_iref) {
tls = msgbuf;
} else {
tls = ((uint8_t *) msgbuf) + XOMP_MSG_FRAME_SIZE;
}
XWI_DEBUG("messaging frame mapped: [%016lx] @ [%016lx]\n", id.base,
(lvaddr_t )msgbuf);
struct bomp_thread_local_data *tlsinfo = malloc(sizeof(*tlsinfo));
tlsinfo->thr = thread_self();
tlsinfo->work = (struct bomp_work *) tls;
tlsinfo->work->data = tlsinfo->work + 1;
g_bomp_state->backend.set_tls(tlsinfo);
#ifdef __k1om__
if (worker_id & XOMP_WID_GATEWAY_FLAG) {
err = xomp_gateway_init();
} else {
if (!svc_iref) {
err = xomp_gateway_bind_svc();
} else {
err = SYS_ERR_OK;
}
}
if (err_is_fail(err)) {
return err;
}
#endif
#ifdef __k1om__
if (!svc_iref) {
err = xeon_phi_client_init(disp_xeon_phi_id());
if (err_is_fail(err)) {
err_push(err, XOMP_ERR_WORKER_INIT_FAILED);
}
xeon_phi_client_set_callbacks(&callbacks);
}
#endif
struct waitset *ws = get_default_waitset();
// XXX: disabling DMA on the host as there is no replication used at this moment
#if XOMP_WORKER_ENABLE_DMA && defined(__k1om__)
/* XXX: use lib numa */
#ifndef __k1om__
uint8_t numanode = 0;
if (disp_get_core_id() > 20) {
numanode = 1;
}
err = dma_manager_wait_for_driver(dma_device_type, numanode);
if (err_is_fail(err)) {
USER_PANIC_ERR(err, "could not wait for the DMA driver");
}
#endif
char svc_name[30];
#ifdef __k1om__
snprintf(svc_name, 30, "%s", XEON_PHI_DMA_SERVICE_NAME);
#else
snprintf(svc_name, 30, "%s.%u", IOAT_DMA_SERVICE_NAME, numanode);
#endif
struct dma_client_info dma_info = {
.type = DMA_CLIENT_INFO_TYPE_NAME,
.device_type = dma_device_type,
.args.name = svc_name
};
err = dma_client_device_init(&dma_info, &dma_dev);
if (err_is_fail(err)) {
USER_PANIC_ERR(err, "DMA device initialization");
}
#endif
if (svc_iref) {
err = xomp_bind(svc_iref, master_bind_cb, NULL, ws,
IDC_EXPORT_FLAGS_DEFAULT);
} else {
struct xomp_frameinfo fi = {
.sendbase = id.base,
.inbuf = ((uint8_t *) msgbuf) + XOMP_MSG_CHAN_SIZE,
.inbufsize = XOMP_MSG_CHAN_SIZE,
.outbuf = ((uint8_t *) msgbuf),
.outbufsize = XOMP_MSG_CHAN_SIZE
};
err = xomp_connect(&fi, master_bind_cb, NULL, ws,
IDC_EXPORT_FLAGS_DEFAULT);
}
if (err_is_fail(err)) {
/* TODO: Clean up */
return err_push(err, XOMP_ERR_WORKER_INIT_FAILED);
}
XWI_DEBUG("Waiting until bound to master...\n");
while (!is_bound) {
messages_wait_and_handle_next();
}
if (xbinding == NULL) {
return XOMP_ERR_WORKER_INIT_FAILED;
}
return SYS_ERR_OK;
}
int main(int argc,
char **argv)
{
errval_t err;
debug_printf("Xeon Phi Test started on the card %u.\n", disp_xeon_phi_id());
debug_printf("Msg Buf Size = %lx, Buf Frame Size = %lx\n",
XPHI_BENCH_MSG_FRAME_SIZE,
XPHI_BENCH_BUF_FRAME_SIZE);
xeon_phi_client_set_callbacks(&callbacks);
err = xeon_phi_client_init(disp_xeon_phi_id());
if (err_is_fail(err)) {
USER_PANIC_ERR(err, "could not init the service\n");
}
err = alloc_local();
assert(err_is_ok(err));
if (disp_xeon_phi_id() == 0) {
char *iface = xeon_phi_domain_build_iface("xeon_phi_inter", 1, 2);
err = xeon_phi_domain_blocking_lookup(iface, &domid);
if (err_is_fail(err)) {
USER_PANIC_ERR(err, "looking up domain id\n");
}
debug_printf("sending open message to %s on node 1\n", iface);
err = xeon_phi_client_chan_open(1, domid, 0xcafebabe, local_frame, 2);
if (err_is_fail(err)) {
USER_PANIC_ERR(err, "could not open channel");
}
}
while (!connected) {
messages_wait_and_handle_next();
}
debug_printf("Initializing UMP channel...\n");
if (disp_xeon_phi_id() != 0) {
err = xeon_phi_client_chan_open(0, domid, 0xdeadbeef, local_frame, 2);
if (err_is_fail(err)) {
USER_PANIC_ERR(err, "could not open channel");
}
} else {
debug_printf("Other node reply: %s\n", (char *) local_buf);
}
err = ump_chan_init(&uc, inbuf,
XPHI_BENCH_MSG_FRAME_SIZE,
outbuf,
XPHI_BENCH_MSG_FRAME_SIZE);
err = ump_chan_init(&uc_rev, inbuf_rev,
XPHI_BENCH_MSG_FRAME_SIZE,
outbuf_rev,
XPHI_BENCH_MSG_FRAME_SIZE);
if (err_is_fail(err)) {
USER_PANIC_ERR(err, "Could not initialize UMP");
}
if (disp_xeon_phi_id() == 1) {
#ifndef XPHI_BENCH_THROUGHPUT
debug_printf("---------------- normal run -----------------\n");
xphi_bench_start_initator_rtt(&bufs, &uc);
debug_printf("---------------- reversed run -----------------\n");
xphi_bench_start_initator_rtt(&bufs_rev, &uc_rev);
#else
#ifdef XPHI_BENCH_SEND_SYNC
debug_printf("---------------- normal run -----------------\n");
xphi_bench_start_initator_sync(&bufs, &uc);
debug_printf("---------------- reversed run -----------------\n");
xphi_bench_start_initator_sync(&bufs_rev, &uc_rev);
#else
debug_printf("---------------- normal run -----------------\n");
xphi_bench_start_initator_async(&bufs, &uc);
debug_printf("---------------- reversed run -----------------\n");
xphi_bench_start_initator_async(&bufs_rev, &uc_rev);
#endif
#endif
} else {
#ifndef XPHI_BENCH_THROUGHPUT
debug_printf("---------------- normal run -----------------\n");
xphi_bench_start_echo(&bufs, &uc);
debug_printf("---------------- reversed run -----------------\n");
xphi_bench_start_echo(&bufs_rev, &uc_rev);
#else
debug_printf("---------------- normal run -----------------\n");
xphi_bench_start_processor(&bufs, &uc);
debug_printf("---------------- reversed run -----------------\n");
xphi_bench_start_processor(&bufs_rev, &uc_rev);
#endif
}
err = dma_manager_wait_for_driver(DMA_DEV_TYPE_XEON_PHI, disp_xeon_phi_id());
if (err_is_fail(err)) {
USER_PANIC_ERR(err, "waiting for drive");
}
struct dma_client_info info = {
.type = DMA_CLIENT_INFO_TYPE_NAME,
.device_type = DMA_DEV_TYPE_XEON_PHI,
.args = {
.name = XEON_PHI_DMA_SERVICE_NAME
}
};
struct dma_client_device *xdev;
err = dma_client_device_init(&info, &xdev);
if (err_is_fail(err)) {
USER_PANIC_ERR(err, "could not initialize client device");
}
struct dma_device *dev = (struct dma_device *) xdev;
err = dma_register_memory((struct dma_device *) dev, local_frame);
if (err_is_fail(err)) {
USER_PANIC_ERR(err, "could not register memory");
}
err = dma_register_memory((struct dma_device *) dev, remote_frame);
if (err_is_fail(err)) {
USER_PANIC_ERR(err, "could not register memory");
}
if (disp_xeon_phi_id() == 1) {
debug_printf("+++++++ DMA / MEMCOPY Benchmark ++++++++\n");
debug_printf("\n");
debug_printf("========================================\n");
debug_printf("\n");
debug_printf("DMA-BENCH: LOCAL -> REMOTE \n");
debug_printf("\n");
debug_printf("========================================\n");
debug_printf("\n");
xphi_bench_memcpy(dev, remote_buf + 2 * XPHI_BENCH_MSG_FRAME_SIZE,
local_buf + 2 * XPHI_BENCH_MSG_FRAME_SIZE,
XPHI_BENCH_BUF_FRAME_SIZE / 2,
remote_base + 2 * XPHI_BENCH_MSG_FRAME_SIZE,
local_base + 2 * XPHI_BENCH_MSG_FRAME_SIZE);
debug_printf("\n");
debug_printf("========================================\n");
debug_printf("\n");
debug_printf("DMA-BENCH: REMOTE -> LOCAL \n");
debug_printf("\n");
debug_printf("========================================\n");
debug_printf("\n");
xphi_bench_memcpy(dev, local_buf + 2 * XPHI_BENCH_MSG_FRAME_SIZE,
remote_buf + 2 * XPHI_BENCH_MSG_FRAME_SIZE,
XPHI_BENCH_BUF_FRAME_SIZE / 2,
local_base + 2 * XPHI_BENCH_MSG_FRAME_SIZE,
remote_base + 2 * XPHI_BENCH_MSG_FRAME_SIZE);
}
debug_printf("benchmark done.");
while (1) {
messages_wait_and_handle_next();
}
}
