static void ms_multiboot_cap_request(struct monitor_binding *b, cslot_t slot) { errval_t err1, err2; struct capref cap = { .cnode = cnode_module, .slot = slot, }; // Call frame_identify to check if cap exists struct frame_identity id; err1 = invoke_frame_identify(cap, &id); if (err_is_fail(err1)) { err2 = b->tx_vtbl.multiboot_cap_reply(b, NOP_CONT, NULL_CAP, err1); } else { err2 = b->tx_vtbl.multiboot_cap_reply(b, NOP_CONT, cap, err1); } if (err_is_fail(err2)) { if (err_no(err2) == FLOUNDER_ERR_TX_BUSY) { struct monitor_state *mon_state = b->st; struct multiboot_cap_state *ms = malloc(sizeof(struct multiboot_cap_state)); assert(ms); ms->slot = slot; ms->elem.cont = ms_multiboot_cap_request_handler; err1 = monitor_enqueue_send(b, &mon_state->queue, get_default_waitset(), &ms->elem.queue); if (err_is_fail(err1)) { USER_PANIC_ERR(err1, "monitor_enqueue_send failed"); } } else { USER_PANIC_ERR(err2, "sending multiboot_cap_reply failed"); } } }
static errval_t map_mmio_space(struct xeon_phi *phi) { errval_t err; void *mmio; struct frame_identity id; err = invoke_frame_identify(mmio_cap, &id); if (err_is_fail(err)) { return err; } err = vspace_map_one_frame(&mmio, (1UL << id.bits), mmio_cap, NULL, NULL); if (err_is_fail(err)) { return err; } XDEBUG("mapped mmio register space @ [%p]\n", mmio); phi->mmio.bits = id.bits; phi->mmio.vbase = (lvaddr_t) mmio; phi->mmio.cap = mmio_cap; phi->mmio.pbase = id.base; phi->mmio.length = (1UL << id.bits); return SYS_ERR_OK; }
static errval_t alloc_local(void) { errval_t err; size_t frame_size = 0; if (disp_xeon_phi_id() == 0) { frame_size = XPHI_BENCH_FRAME_SIZE_HOST; } else { frame_size = XPHI_BENCH_FRAME_SIZE_CARD; } if (!frame_size) { frame_size = 4096; } debug_printf("Allocating a frame of size: %lx\n", frame_size); size_t alloced_size = 0; err = frame_alloc(&local_frame, frame_size, &alloced_size); assert(err_is_ok(err)); assert(alloced_size >= frame_size); struct frame_identity id; err = invoke_frame_identify(local_frame, &id); assert(err_is_ok(err)); local_base = id.base; local_frame_sz = alloced_size; err = vspace_map_one_frame(&local_buf, alloced_size, local_frame, NULL, NULL); return err; }
void alloc_local(void) { errval_t err; #ifndef __k1om__ uint64_t minbase, maxlimit; ram_get_affinity(&minbase, &maxlimit); ram_set_affinity(XPHI_BENCH_RAM_MINBASE, XPHI_BENCH_RAM_MAXLIMIT); #endif size_t alloced_size = 0; err = frame_alloc(&local_frame, XPHI_BENCH_MSG_FRAME_SIZE, &alloced_size); EXPECT_SUCCESS(err, "frame_alloc"); #ifndef __k1om__ ram_set_affinity(minbase, maxlimit); #endif struct frame_identity id; err = invoke_frame_identify(local_frame, &id); EXPECT_SUCCESS(err, "invoke_frame_identify"); local_base = id.base; local_frame_sz = alloced_size; debug_printf("alloc_local | Frame base: %016lx, size=%lx\n", id.base, 1UL << id.bits); err = vspace_map_one_frame(&local_buf, alloced_size, local_frame, NULL, NULL); EXPECT_SUCCESS(err, "vspace_map_one_frame"); }
/** * \brief initializes a dma descriptor ring and allocates memory for it * * \param ring the ring structure to initialize * \param size number of elements in the ring * * \returns SYS_ERR_OK on success * errval on error */ errval_t xeon_phi_dma_desc_ring_alloc(struct xdma_ring *ring, uint16_t size) { errval_t err; memset(ring, 0, sizeof(*ring)); assert(size < (XEON_PHI_DMA_DESC_RING_MAX)); assert(IS_POW2(size)); #ifndef __k1om__ /* * we set the ram affinity to the maximum range mapped by the system memory * page tables when being on the host. Otherwise the card cannot access it. */ uint64_t minbase, maxlimit; ram_get_affinity(&minbase, &maxlimit); ram_set_affinity(0, XEON_PHI_SYSMEM_SIZE-8*XEON_PHI_SYSMEM_PAGE_SIZE); #endif size_t frame_size = ((size_t) size) * XEON_PHI_DMA_DESC_SIZE; err = frame_alloc(&ring->cap, frame_size, NULL); #ifndef __k1om__ ram_set_affinity(minbase, maxlimit); #endif if (err_is_fail(err)) { return err; } err = vspace_map_one_frame_attr(&ring->vbase, frame_size, ring->cap, VREGION_FLAGS_READ_WRITE, NULL, NULL); if (err_is_fail(err)) { cap_destroy(ring->cap); return err; } struct frame_identity id; err = invoke_frame_identify(ring->cap, &id); assert(err_is_ok(err)); ring->pbase = id.base; ring->size = size; memset(ring->vbase, 0, frame_size); return SYS_ERR_OK; }
// Callback from device manager void qd_queue_init_data(struct e10k_binding *b, struct capref registers, uint64_t macaddr) { struct frame_identity frameid = { .base = 0, .bits = 0 }; errval_t err; void *virt; INITDEBUG("idc_queue_init_data\n"); mac_address = macaddr; // Map device registers invoke_frame_identify(registers, &frameid); err = vspace_map_one_frame_attr(&virt, 1 << frameid.bits, registers, VREGION_FLAGS_READ_WRITE_NOCACHE, NULL, NULL); assert(err_is_ok(err)); // Initialize mackerel device d = malloc(sizeof(*d)); #ifndef VF e10k_initialize(d, virt); #else e10k_vf_initialize(d, virt); #endif // Initialize queue setup_queue(); } // Callback from device manager void qd_queue_memory_registered(struct e10k_binding *b) { initialized = 1; hwqueue_initialized(); // Register queue with queue_mgr library #ifndef LIBRARY ethersrv_init((char*) service_name, qi, get_mac_addr_fn, terminate_queue_fn, transmit_pbuf_list_fn, find_tx_free_slot_count_fn, handle_free_tx_slot_fn, RXBUFSZ, register_rx_buffer_fn, find_rx_free_slot_count_fn); #else ethernetif_backend_init((char*) service_name, qi, get_mac_addr_fn, terminate_queue_fn, transmit_pbuf_list_fn, find_tx_free_slot_count_fn, handle_free_tx_slot_fn, RXBUFSZ, register_rx_buffer_fn, find_rx_free_slot_count_fn); #endif }
/** * \brief Initialise a new UMP channel to accept an incoming binding request * * \param uc Storage for channel state * \param mon_id Monitor's connection ID for this channel * \param frame Frame capability containing channel * \param inchanlen Size of incoming channel, in bytes (multiple of #UMP_MSG_BYTES) * \param outchanlen Size of outgoing channel, in bytes (multiple of #UMP_MSG_BYTES) */ errval_t ump_chan_accept(struct ump_chan *uc, uintptr_t mon_id, struct capref frame, size_t inchanlen, size_t outchanlen) { errval_t err; uc->monitor_id = mon_id; uc->frame = frame; // check that the frame is big enough struct frame_identity frameid; err = invoke_frame_identify(frame, &frameid); if (err_is_fail(err)) { return err_push(err, LIB_ERR_FRAME_IDENTIFY); } // Ids for tracing uc->recvid = (uintptr_t)(frameid.base + outchanlen); uc->sendid = (uintptr_t)frameid.base; size_t framesize = ((uintptr_t)1) << frameid.bits; if (framesize < inchanlen + outchanlen) { return LIB_ERR_UMP_FRAME_OVERFLOW; } // map it in void *buf; err = vspace_map_one_frame_attr(&buf, framesize, frame, UMP_MAP_ATTR, NULL, &uc->vregion); if (err_is_fail(err)) { cap_destroy(uc->frame); return err_push(err, LIB_ERR_VSPACE_MAP); } // initialise channel state err = ump_chan_init(uc, (char *)buf + outchanlen, inchanlen, buf, outchanlen); if (err_is_fail(err)) { vregion_destroy(uc->vregion); cap_destroy(uc->frame); return err; } /* mark connected */ uc->connstate = UMP_CONNECTED; return SYS_ERR_OK; }
static errval_t msg_open_cb(xphi_dom_id_t domain, uint64_t usrdata, struct capref msgframe, uint8_t type) { errval_t err; domainid = domain; struct frame_identity id; err = invoke_frame_identify(msgframe, &id); EXPECT_SUCCESS(err, "frame identify"); debug_printf("msg_open_cb | Frame base: %016lx, size=%lx\n", id.base, 1UL << id.bits); assert((1UL << id.bits) >= XPHI_BENCH_MSG_FRAME_SIZE); err = vspace_map_one_frame(&remote_buf, XPHI_BENCH_MSG_FRAME_SIZE, msgframe, NULL, NULL); EXPECT_SUCCESS(err, "vspace map frame"); remote_frame = msgframe; remote_base = id.base; remote_frame_sz = (1UL << id.bits); init_buffer(); connected = 0x1; debug_printf("Initializing UMP channel...\n"); err = ump_chan_init(&xphi_uc, inbuf, XPHI_BENCH_MSG_CHAN_SIZE, outbuf, XPHI_BENCH_MSG_CHAN_SIZE); EXPECT_SUCCESS(err, "initialize ump channel"); err = ump_chan_init(&xphi_uc_rev, inbuf_rev, XPHI_BENCH_MSG_CHAN_SIZE, outbuf_rev, XPHI_BENCH_MSG_CHAN_SIZE); EXPECT_SUCCESS(err, "initialize ump channel"); return SYS_ERR_OK; }
static void get_bootinfo(struct monitor_blocking_binding *b) { errval_t err; struct capref frame = { .cnode = cnode_task, .slot = TASKCN_SLOT_BOOTINFO }; struct frame_identity id = { .base = 0, .bits = 0 }; err = invoke_frame_identify(frame, &id); assert(err_is_ok(err)); err = b->tx_vtbl.get_bootinfo_response(b, NOP_CONT, SYS_ERR_OK, frame, (size_t)1 << id.bits); if (err_is_fail(err)) { if (err_no(err) == FLOUNDER_ERR_TX_BUSY) { err = b->register_send(b, get_default_waitset(), MKCONT((void (*)(void *))get_bootinfo, b)); if (err_is_fail(err)) { USER_PANIC_ERR(err, "register_send failed"); } } USER_PANIC_ERR(err, "sending get_bootinfo_response failed"); } } /* ----------------------- BOOTINFO REQUEST CODE END ----------------------- */ static void get_ipi_cap(struct monitor_blocking_binding *b) { errval_t err; // XXX: We should not just hand out this cap to everyone // who requests it. There is currently no way to determine // if the client is a valid recipient err = b->tx_vtbl.get_ipi_cap_response(b, NOP_CONT, cap_ipi); assert(err_is_ok(err)); }
// populates the given buffer with given capref static errval_t populate_buffer(struct buffer_descriptor *buffer, struct capref cap) { buffer->cap = cap; struct frame_identity pa; errval_t err = invoke_frame_identify(cap, &pa); if (!err_is_ok(err)) { printf("invoke_frame_identify failed\n"); abort(); } buffer->pa = pa.base; buffer->bytes = pa.bytes; err = vspace_map_one_frame(&buffer->va, buffer->bytes, cap, NULL, NULL); /* err = vspace_map_one_frame_attr(&buffer->va, (1L << buffer->bits), cap, VREGION_FLAGS_READ_WRITE_NOCACHE, NULL, NULL); */ if (err_is_fail(err)) { DEBUG_ERR(err, "vspace_map_one_frame failed"); // FIXME: report more sensible error return(ETHERSRV_ERR_TOO_MANY_BUFFERS); } netd_buffer_count++; buffer_id_counter++; buffer->buffer_id = buffer_id_counter; // printf("### buffer gets id %"PRIu64"\n", buffer->buffer_id); if (buffer->buffer_id == 3) { first_app_b = buffer; } buffer->next = buffers_list; // Adding the buffer on the top of buffer list. // buffers_list = buffer; return SYS_ERR_OK; } // end function: populate_buffer
static errval_t msg_open_cb(xphi_dom_id_t domain, uint64_t usrdata, struct capref msgframe, uint8_t type) { errval_t err; domid = domain; struct frame_identity id; err = invoke_frame_identify(msgframe, &id); if (err_is_fail(err)) { USER_PANIC_ERR(err, "could not identify the frame"); } debug_printf("msg_open_cb | Frame base: %016lx, size=%lx, ud:%lx\n", id.base, 1UL << id.bits, usrdata); remote_frame = msgframe; remote_base = id.base; remote_frame_sz = (1UL << id.bits); err = vspace_map_one_frame(&remote_buf, remote_frame_sz, msgframe, NULL, NULL); if (err_is_fail(err)) { USER_PANIC_ERR(err, "Could not map the frame"); } init_buffer_c0(); connected = 0x1; return SYS_ERR_OK; }
/** * \brief allocates and maps a memory region to be used for DMA purposes * * \param bytes minimum size of the memory region in bytes * \param flags VREGION flags how the region gets mapped * \param mem returns the mapping information * * \returns SYS_ERR_OK on success * errval on error */ errval_t dma_mem_alloc(size_t bytes, vregion_flags_t flags, struct dma_mem *mem) { errval_t err; if (mem == NULL) { return DMA_ERR_ARG_INVALID; } err = frame_alloc(&mem->frame, bytes, &mem->bytes); if (err_is_fail(err)) { return err; } struct frame_identity id; err = invoke_frame_identify(mem->frame, &id); if (err_is_fail(err)) { dma_mem_free(mem); return err; } mem->paddr = id.base; void *addr; err = vspace_map_one_frame_attr(&addr, mem->bytes, mem->frame, flags, NULL, NULL); if (err_is_fail(err)) { dma_mem_free(mem); return err; } mem->vaddr = (lvaddr_t)addr; return SYS_ERR_OK; }
/** * \brief Notification of a newly booted monitor. * Setup our connection and request the sender to proxy * the bind request to the monitor */ static void new_monitor_notify(struct intermon_binding *b, coreid_t core_id) { errval_t err; /* Setup the connection */ ram_set_affinity(SHARED_MEM_MIN + (PERCORE_MEM_SIZE * my_core_id), SHARED_MEM_MIN + (PERCORE_MEM_SIZE * (my_core_id + 1))); struct capref frame; err = frame_alloc(&frame, MON_URPC_SIZE, NULL); if (err_is_fail(err)) { DEBUG_ERR(err, "frame_alloc failed"); return; // FIXME: cleanup } ram_set_affinity(0, 0); // Reset affinity void *buf; err = vspace_map_one_frame_attr(&buf, MON_URPC_SIZE, frame, VREGION_FLAGS_READ_WRITE_NOCACHE, NULL, NULL); if (err_is_fail(err)) { DEBUG_ERR(err, "vspace_map_one_frame failed"); assert(buf); // XXX } // XXX: Clear the frame (kernel can't do it for us) memset(buf, 0, MON_URPC_SIZE); // Allocate my own notification caps struct capref ep, my_notify_cap; struct lmp_endpoint *iep; int chanid; err = endpoint_create(LMP_RECV_LENGTH, &ep, &iep); assert(err_is_ok(err)); err = notification_allocate(ep, &chanid); assert(err == SYS_ERR_OK); err = notification_create_cap(chanid, my_core_id, &my_notify_cap); assert(err == SYS_ERR_OK); // init our end of the binding and channel struct intermon_ump_ipi_binding *ump_binding = malloc(sizeof(struct intermon_ump_ipi_binding)); assert(ump_binding != NULL); err = intermon_ump_ipi_init(ump_binding, get_default_waitset(), buf, MON_URPC_CHANNEL_LEN, buf + MON_URPC_CHANNEL_LEN, MON_URPC_CHANNEL_LEN, NULL_CAP, my_notify_cap, ep, iep); assert(err_is_ok(err)); /* if (err_is_fail(err)) { */ /* cap_destroy(frame); */ /* return err_push(err, LIB_ERR_UMP_CHAN_BIND); */ /* } */ // Identify UMP frame for tracing struct frame_identity umpid = { .base = 0, .bits = 0 }; err = invoke_frame_identify(frame, &umpid); assert(err_is_ok(err)); ump_binding->ump_state.chan.recvid = (uintptr_t)umpid.base; ump_binding->ump_state.chan.sendid = (uintptr_t)(umpid.base + MON_URPC_CHANNEL_LEN); err = intermon_init(&ump_binding->b, core_id); assert(err_is_ok(err)); /* Identify the frame cap */ struct capability frame_cap; err = monitor_cap_identify(frame, &frame_cap); if (err_is_fail(err)) { DEBUG_ERR(err, "monitor_cap_identify failed"); return; // FIXME: cleanup } intermon_caprep_t caprep; capability_to_caprep(&frame_cap, &caprep); /* reply to the sending monitor to proxy request */ err = b->tx_vtbl.bind_monitor_proxy_scc(b, NOP_CONT, core_id, caprep, chanid, my_core_id); if (err_is_fail(err)) { DEBUG_ERR(err, "bind proxy request failed"); } } errval_t arch_intermon_init(struct intermon_binding *b) { b->rx_vtbl.bind_monitor_request_scc = bind_monitor_request_scc; b->rx_vtbl.bind_monitor_reply_scc = bind_monitor_reply_scc; b->rx_vtbl.bind_monitor_proxy_scc = bind_monitor_proxy_scc; b->rx_vtbl.new_monitor_notify = new_monitor_notify; return SYS_ERR_OK; }
errval_t spawn_xcore_monitor(coreid_t coreid, int hwid, enum cpu_type cpu_type, const char *cmdline, struct frame_identity urpc_frame_id, struct capref kcb) { uint64_t start = 0; const char *monitorname = NULL, *cpuname = NULL; genpaddr_t arch_page_size; errval_t err; err = get_architecture_config(cpu_type, &arch_page_size, &monitorname, &cpuname); assert(err_is_ok(err)); DEBUG("loading kernel: %s\n", cpuname); DEBUG("loading 1st app: %s\n", monitorname); // compute size of frame needed and allocate it DEBUG("%s:%s:%d: urpc_frame_id.base=%"PRIxGENPADDR"\n", __FILE__, __FUNCTION__, __LINE__, urpc_frame_id.base); DEBUG("%s:%s:%d: urpc_frame_id.size=%d\n", __FILE__, __FUNCTION__, __LINE__, urpc_frame_id.bits); if (benchmark_flag) { start = bench_tsc(); } static size_t cpu_binary_size; static lvaddr_t cpu_binary = 0; static genpaddr_t cpu_binary_phys; static const char* cached_cpuname = NULL; if (cpu_binary == 0) { cached_cpuname = cpuname; // XXX: Caching these for now, until we have unmap err = lookup_module(cpuname, &cpu_binary, &cpu_binary_phys, &cpu_binary_size); if (err_is_fail(err)) { DEBUG_ERR(err, "Can not lookup module"); return err; } } // Ensure caching actually works and we're // always loading same binary. If this starts to fail, get rid of caching. assert (strcmp(cached_cpuname, cpuname) == 0); static size_t monitor_binary_size; static lvaddr_t monitor_binary = 0; static genpaddr_t monitor_binary_phys; static const char* cached_monitorname = NULL; if (monitor_binary == 0) { cached_monitorname = monitorname; // XXX: Caching these for now, until we have unmap err = lookup_module(monitorname, &monitor_binary, &monitor_binary_phys, &monitor_binary_size); if (err_is_fail(err)) { DEBUG_ERR(err, "Can not lookup module"); return err; } } // Again, ensure caching actually worked (see above) assert (strcmp(cached_monitorname, monitorname) == 0); if (benchmark_flag) { bench_data->load = bench_tsc() - start; start = bench_tsc(); } struct capref cpu_memory_cap; struct frame_identity frameid; size_t cpu_memory; err = allocate_kernel_memory(cpu_binary, arch_page_size, &cpu_memory_cap, &cpu_memory, &frameid); if (err_is_fail(err)) { DEBUG_ERR(err, "Can not allocate space for new app kernel."); return err; } err = cap_mark_remote(cpu_memory_cap); if (err_is_fail(err)) { DEBUG_ERR(err, "Can not mark cap remote."); return err; } void *cpu_buf_memory; err = vspace_map_one_frame(&cpu_buf_memory, cpu_memory, cpu_memory_cap, NULL, NULL); if (err_is_fail(err)) { return err_push(err, LIB_ERR_VSPACE_MAP); } if (benchmark_flag) { bench_data->alloc_cpu = bench_tsc() - start; start = bench_tsc(); } /* Chunk of memory to load monitor on the app core */ struct capref spawn_memory_cap; struct frame_identity spawn_memory_identity; err = frame_alloc_identify(&spawn_memory_cap, X86_CORE_DATA_PAGES * arch_page_size, NULL, &spawn_memory_identity); if (err_is_fail(err)) { return err_push(err, LIB_ERR_FRAME_ALLOC); } err = cap_mark_remote(spawn_memory_cap); if (err_is_fail(err)) { DEBUG_ERR(err, "Can not mark cap remote."); return err; } if (benchmark_flag) { bench_data->alloc_mon = bench_tsc() - start; start = bench_tsc(); } /* Load cpu */ struct elf_allocate_state state; state.vbase = (char *)cpu_buf_memory + arch_page_size; assert(sizeof(struct x86_core_data) <= arch_page_size); state.elfbase = elf_virtual_base(cpu_binary); struct Elf64_Ehdr *cpu_head = (struct Elf64_Ehdr *)cpu_binary; genvaddr_t cpu_entry; err = elf_load(cpu_head->e_machine, elfload_allocate, &state, cpu_binary, cpu_binary_size, &cpu_entry); if (err_is_fail(err)) { return err; } if (benchmark_flag) { bench_data->elf_load = bench_tsc() - start; start = bench_tsc(); } err = relocate_cpu_binary(cpu_binary, cpu_head, state, frameid, arch_page_size); if (err_is_fail(err)) { DEBUG_ERR(err, "Can not relocate new kernel."); return err; } if (benchmark_flag) { bench_data->elf_reloc = bench_tsc() - start; } genvaddr_t cpu_reloc_entry = cpu_entry - state.elfbase + frameid.base + arch_page_size; /* Compute entry point in the foreign address space */ forvaddr_t foreign_cpu_reloc_entry = (forvaddr_t)cpu_reloc_entry; /* Setup the core_data struct in the new kernel */ struct x86_core_data *core_data = (struct x86_core_data *)cpu_buf_memory; switch (cpu_head->e_machine) { case EM_X86_64: case EM_K1OM: core_data->elf.size = sizeof(struct Elf64_Shdr); core_data->elf.addr = cpu_binary_phys + (uintptr_t)cpu_head->e_shoff; core_data->elf.num = cpu_head->e_shnum; break; case EM_386: core_data->elf.size = sizeof(struct Elf32_Shdr); struct Elf32_Ehdr *head32 = (struct Elf32_Ehdr *)cpu_binary; core_data->elf.addr = cpu_binary_phys + (uintptr_t)head32->e_shoff; core_data->elf.num = head32->e_shnum; break; default: return SPAWN_ERR_UNKNOWN_TARGET_ARCH; } core_data->module_start = cpu_binary_phys; core_data->module_end = cpu_binary_phys + cpu_binary_size; core_data->urpc_frame_base = urpc_frame_id.base; core_data->urpc_frame_bits = urpc_frame_id.bits; core_data->monitor_binary = monitor_binary_phys; core_data->monitor_binary_size = monitor_binary_size; core_data->memory_base_start = spawn_memory_identity.base; core_data->memory_bits = spawn_memory_identity.bits; core_data->src_core_id = disp_get_core_id(); core_data->src_arch_id = my_arch_id; core_data->dst_core_id = coreid; struct frame_identity fid; err = invoke_frame_identify(kcb, &fid); if (err_is_fail(err)) { USER_PANIC_ERR(err, "Invoke frame identity for KCB failed. " "Did you add the syscall handler for that architecture?"); } DEBUG("%s:%s:%d: fid.base is 0x%"PRIxGENPADDR"\n", __FILE__, __FUNCTION__, __LINE__, fid.base); core_data->kcb = (genpaddr_t) fid.base; #ifdef CONFIG_FLOUNDER_BACKEND_UMP_IPI core_data->chan_id = chanid; #endif if (cmdline != NULL) { // copy as much of command line as will fit snprintf(core_data->kernel_cmdline, sizeof(core_data->kernel_cmdline), "%s %s", cpuname, cmdline); // ensure termination core_data->kernel_cmdline[sizeof(core_data->kernel_cmdline) - 1] = '\0'; DEBUG("%s:%s:%d: %s\n", __FILE__, __FUNCTION__, __LINE__, core_data->kernel_cmdline); } /* Invoke kernel capability to boot new core */ if (cpu_type == CPU_X86_64 || cpu_type == CPU_K1OM) { start_aps_x86_64_start(hwid, foreign_cpu_reloc_entry); } #ifndef __k1om__ else if (cpu_type == CPU_X86_32) { start_aps_x86_32_start(hwid, foreign_cpu_reloc_entry); } #endif /* Clean up */ // XXX: Should not delete the remote caps? err = cap_destroy(spawn_memory_cap); if (err_is_fail(err)) { USER_PANIC_ERR(err, "cap_destroy failed"); } err = vspace_unmap(cpu_buf_memory); if (err_is_fail(err)) { USER_PANIC_ERR(err, "vspace unmap CPU driver memory failed"); } err = cap_destroy(cpu_memory_cap); if (err_is_fail(err)) { USER_PANIC_ERR(err, "cap_destroy failed"); } return SYS_ERR_OK; }
static errval_t spawn_setup_argcn(struct spawninfo *si, struct capref argumentcn_cap) { errval_t err; if (capref_is_null(argumentcn_cap)) { return SYS_ERR_OK; } struct capref dest = { .cnode = si->rootcn, .slot = ROOTCN_SLOT_ARGCN }; err = cap_copy(dest, argumentcn_cap); if (err_is_fail(err)) { return err_push(err, SPAWN_ERR_COPY_ARGCN); } return SYS_ERR_OK; } /** * \brief Load an image * * \param si Struct used by the library * \param binary The image to load * \param type The type of arch to load for * \param name Name of the image required only to place it in disp * struct * \param coreid Coreid to load for, required only to place it in disp * struct * \param argv Command-line arguments, NULL-terminated * \param envp Environment, NULL-terminated * \param inheritcn_cap Cap to a CNode containing capabilities to be inherited * \param argcn_cap Cap to a CNode containing capabilities passed as * arguments */ errval_t spawn_load_image(struct spawninfo *si, lvaddr_t binary, size_t binary_size, enum cpu_type type, const char *name, coreid_t coreid, char *const argv[], char *const envp[], struct capref inheritcn_cap, struct capref argcn_cap) { errval_t err; si->cpu_type = type; /* Initialize cspace */ err = spawn_setup_cspace(si); if (err_is_fail(err)) { return err_push(err, SPAWN_ERR_SETUP_CSPACE); } /* Initialize vspace */ err = spawn_setup_vspace(si); if (err_is_fail(err)) { return err_push(err, SPAWN_ERR_VSPACE_INIT); } si->name = name; genvaddr_t entry; void* arch_info; /* Load the image */ err = spawn_arch_load(si, binary, binary_size, &entry, &arch_info); if (err_is_fail(err)) { return err_push(err, SPAWN_ERR_LOAD); } /* Setup dispatcher frame */ err = spawn_setup_dispatcher(si, coreid, name, entry, arch_info); if (err_is_fail(err)) { return err_push(err, SPAWN_ERR_SETUP_DISPATCHER); } /* Setup inherited caps */ err = spawn_setup_inherited_caps(si, inheritcn_cap); if (err_is_fail(err)) { return err_push(err, SPAWN_ERR_SETUP_INHERITED_CAPS); } /* Setup argument caps */ err = spawn_setup_argcn(si, argcn_cap); if (err_is_fail(err)) { return err_push(err, SPAWN_ERR_SETUP_ARGCN); } // Add vspace-pspace mapping to environment char envstr[2048]; #ifdef __x86__ // SK: si->vregions only valid on x86 snprintf(envstr, 2048, "ARRAKIS_PMAP="); for(int i = 0; i < si->vregions; i++) { struct memobj_anon *m = (struct memobj_anon *)si->vregion[i]->memobj; assert(m->m.type == ANONYMOUS); for(struct memobj_frame_list *f = m->frame_list; f != NULL; f = f->next) { struct frame_identity id; err = invoke_frame_identify(f->frame, &id); assert(err_is_ok(err)); char str[128]; snprintf(str, 128, "%" PRIxGENVADDR ":%" PRIxGENPADDR ":%zx ", si->base[i] + f->offset, id.base, f->size); strcat(envstr, str); } } #endif /* __x86__ */ char **myenv = (char **)envp; for(int i = 0; i < MAX_ENVIRON_VARS; i++) { if(i + 1 == MAX_ENVIRON_VARS) { printf("spawnd: Couldn't set environemnt. Out of variables!\n"); abort(); } if(myenv[i] == NULL) { myenv[i] = envstr; myenv[i+1] = NULL; break; } } /* Setup cmdline args */ err = spawn_setup_env(si, argv, envp); if (err_is_fail(err)) { return err_push(err, SPAWN_ERR_SETUP_ENV); } return SYS_ERR_OK; }
static void add_memory_call_rx(struct xomp_binding *b, struct capref frame, uint64_t addr, uint8_t type) { XWI_DEBUG("add_memory_call_rx: addr:%lx, tyep: %u\n", addr, type); struct txq_msg_st *msg_st = txq_msg_st_alloc(&txq); assert(msg_st != NULL); msg_st->send = add_memory_response_tx; msg_st->cleanup = NULL; uint32_t map_flags = 0x0; switch ((xomp_frame_type_t) type) { case XOMP_FRAME_TYPE_MSG: map_flags = VREGION_FLAGS_READ_WRITE; break; case XOMP_FRAME_TYPE_SHARED_RW: map_flags = VREGION_FLAGS_READ_WRITE; break; case XOMP_FRAME_TYPE_SHARED_RO: map_flags = VREGION_FLAGS_READ; break; default: USER_PANIC("unknown type: %u", type) break; } struct frame_identity id; msg_st->err = invoke_frame_identify(frame, &id); if(err_is_fail(msg_st->err)) { txq_send(msg_st); return; } #if XOMP_WORKER_ENABLE_DMA if (0) { // todo: replicate frame on the same node if needed.. replicate_frame(addr, &frame); } #endif #if XOMP_BENCH_WORKER_EN cycles_t map_start = bench_tsc(); #endif if (addr) { msg_st->err = vspace_map_one_frame_fixed_attr(addr, (1UL << id.bits), frame, map_flags, NULL, NULL); } else { void *map_addr; msg_st->err = vspace_map_one_frame_attr(&map_addr, (1UL << id.bits), frame, map_flags, NULL, NULL); } #if XOMP_BENCH_WORKER_EN cycles_t timer_end = bench_tsc(); debug_printf("%lx mem map %016lx took %lu cycles, %lu ms\n", worker_id, addr, timer_end - map_start, bench_tsc_to_ms(timer_end - map_start)); #endif txq_send(msg_st); }
/** * \brief Initialise a new UMP channel and initiate a binding * * \param uc Storage for channel state * \param cont Continuation for bind completion/failure * \param qnode Storage for an event queue node (used for queuing bind request) * \param iref IREF to which to bind * \param monitor_binding Monitor binding to use * \param inchanlen Size of incoming channel, in bytes (rounded to #UMP_MSG_BYTES) * \param outchanlen Size of outgoing channel, in bytes (rounded to #UMP_MSG_BYTES) * \param notify_cap Capability to use for notifications, or #NULL_CAP */ errval_t ump_chan_bind(struct ump_chan *uc, struct ump_bind_continuation cont, struct event_queue_node *qnode, iref_t iref, struct monitor_binding *monitor_binding, size_t inchanlen, size_t outchanlen, struct capref notify_cap) { errval_t err; // round up channel sizes to message size inchanlen = ROUND_UP(inchanlen, UMP_MSG_BYTES); outchanlen = ROUND_UP(outchanlen, UMP_MSG_BYTES); // compute size of frame needed and allocate it size_t framesize = inchanlen + outchanlen; err = frame_alloc(&uc->frame, framesize, &framesize); if (err_is_fail(err)) { return err_push(err, LIB_ERR_FRAME_ALLOC); } // map it in void *buf; err = vspace_map_one_frame_attr(&buf, framesize, uc->frame, UMP_MAP_ATTR, NULL, &uc->vregion); if (err_is_fail(err)) { cap_destroy(uc->frame); return err_push(err, LIB_ERR_VSPACE_MAP); } // initialise channel state err = ump_chan_init(uc, buf, inchanlen, (char *)buf + inchanlen, outchanlen); if (err_is_fail(err)) { vregion_destroy(uc->vregion); cap_destroy(uc->frame); return err; } // Ids for tracing struct frame_identity id; err = invoke_frame_identify(uc->frame, &id); if (err_is_fail(err)) { vregion_destroy(uc->vregion); cap_destroy(uc->frame); return err_push(err, LIB_ERR_FRAME_IDENTIFY); } uc->recvid = (uintptr_t)id.base; uc->sendid = (uintptr_t)(id.base + inchanlen); // store bind args uc->bind_continuation = cont; uc->monitor_binding = monitor_binding; uc->iref = iref; uc->inchanlen = inchanlen; uc->outchanlen = outchanlen; uc->notify_cap = notify_cap; // wait for the ability to use the monitor binding uc->connstate = UMP_BIND_WAIT; event_mutex_enqueue_lock(&monitor_binding->mutex, qnode, MKCLOSURE(send_bind_cont, uc)); return SYS_ERR_OK; }
/// Map in the frame caps for a module into our vspace, return their location errval_t spawn_map_module(struct mem_region *module, size_t *retsize, lvaddr_t *retaddr, genpaddr_t *retpaddr) { assert(module != NULL); assert(module->mr_type == RegionType_Module); errval_t err; size_t size = module->mrmod_size; void *base; struct memobj *memobj; struct vregion *vregion; err = vspace_map_anon_attr(&base, &memobj, &vregion, size, &size, VREGION_FLAGS_READ); if (err_is_fail(err)) { return err_push(err, LIB_ERR_VSPACE_MAP); } struct capref frame = { .cnode = cnode_module, .slot = module->mrmod_slot, }; if (retpaddr != NULL) { *retpaddr = module->mr_base; } if (retsize != NULL) { *retsize = size; } if (retaddr != NULL) { *retaddr = (lvaddr_t)base; } size_t offset = 0; while (size > 0) { assert((size & BASE_PAGE_MASK) == 0); struct frame_identity id; err = invoke_frame_identify(frame, &id); assert(err_is_ok(err)); err = memobj->f.fill(memobj, offset, frame, 1UL << id.bits); if (err_is_fail(err)) { return err_push(err, LIB_ERR_MEMOBJ_FILL); } err = memobj->f.pagefault(memobj, vregion, offset, 0); if (err_is_fail(err)) { return err_push(err, LIB_ERR_MEMOBJ_PAGEFAULT_HANDLER); } frame.slot ++; size -= (1UL << id.bits); offset += (1UL << id.bits); } return SYS_ERR_OK; } errval_t spawn_unmap_module(lvaddr_t mapped_addr) { return vspace_unmap((void *)mapped_addr); } /// Returns a raw pointer to the modules string area string const char *multiboot_module_rawstring(struct mem_region *region) { if (multiboot_strings == NULL) { errval_t err; /* Map in multiboot module strings area */ struct capref mmstrings_cap = { .cnode = cnode_module, .slot = 0 }; err = vspace_map_one_frame_attr((void**)&multiboot_strings, BASE_PAGE_SIZE, mmstrings_cap, VREGION_FLAGS_READ, NULL, NULL); if (err_is_fail(err)) { DEBUG_ERR(err, "vspace_map failed"); return NULL; } #if 0 printf("Mapped multiboot_strings at %p\n", multiboot_strings); for (int i = 0; i < 256; i++) { if ((i & 15) == 0) printf("%04x ", i); printf ("%02x ", multiboot_strings[i]& 0xff); if ((i & 15) == 15) printf("\n"); } #endif } if (region == NULL || region->mr_type != RegionType_Module) { return NULL; } return multiboot_strings + region->mrmod_data; } errval_t multiboot_cleanup_mapping(void) { errval_t err = vspace_unmap(multiboot_strings); if (err_is_fail(err)) { DEBUG_ERR(err, "multiboot_cleanup_mapping: vspace_unmap() failed\n"); return err_push(err, LIB_ERR_VSPACE_REMOVE_REGION); } multiboot_strings = NULL; return SYS_ERR_OK; }
errval_t vspace_mmu_aware_reset(struct vspace_mmu_aware *state, struct capref frame, size_t size) { errval_t err; struct vregion *vregion; struct capref oldframe; void *vbuf; // create copy of new region err = slot_alloc(&oldframe); if (err_is_fail(err)) { return err; } err = cap_copy(oldframe, frame); if (err_is_fail(err)) { return err; } err = vspace_map_one_frame_attr_aligned(&vbuf, size, oldframe, VREGION_FLAGS_READ_WRITE | VREGION_FLAGS_LARGE, LARGE_PAGE_SIZE, NULL, &vregion); if (err_is_fail(err)) { return err; } // copy over data to new frame genvaddr_t gen_base = vregion_get_base_addr(&state->vregion); memcpy(vbuf, (void*)(lvaddr_t)gen_base, state->mapoffset); err = vregion_destroy(vregion); if (err_is_fail(err)) { return err; } genvaddr_t offset = 0; // Unmap backing frames for [0, size) in state.vregion do { err = state->memobj.m.f.unfill(&state->memobj.m, 0, &oldframe, &offset); if (err_is_fail(err) && err_no(err) != LIB_ERR_MEMOBJ_UNFILL_TOO_HIGH_OFFSET) { return err_push(err, LIB_ERR_MEMOBJ_UNMAP_REGION); } struct frame_identity fi; // increase address err = invoke_frame_identify(oldframe, &fi); if (err_is_fail(err)) { return err; } offset += (1UL<<fi.bits); err = cap_destroy(oldframe); if (err_is_fail(err)) { return err; } } while(offset < state->mapoffset); // Map new frame in err = state->memobj.m.f.fill(&state->memobj.m, 0, frame, size); if (err_is_fail(err)) { return err_push(err, LIB_ERR_MEMOBJ_FILL); } err = state->memobj.m.f.pagefault(&state->memobj.m, &state->vregion, 0, 0); if (err_is_fail(err)) { return err_push(err, LIB_ERR_MEMOBJ_PAGEFAULT_HANDLER); } state->mapoffset = size; return SYS_ERR_OK; }
static void impl_test(void) { errval_t err; debug_printf("Doing an implementation test\n"); struct capref frame; err = frame_alloc(&frame, 2 * BUFFER_SIZE, NULL); assert(err_is_ok(err)); struct frame_identity id; err = invoke_frame_identify(frame, &id); assert(err_is_ok(err)); void *buf; err = vspace_map_one_frame(&buf, 1UL << id.bits, frame, NULL, NULL); assert(err_is_ok(err)); memset(buf, 0, 1UL << id.bits); memset(buf, 0xA5, BUFFER_SIZE); struct ioat_dma_req_setup setup = { .type = IOAT_DMA_REQ_TYPE_MEMCPY, .src = id.base, .dst = id.base + BUFFER_SIZE, .bytes = BUFFER_SIZE, .done_cb = impl_test_cb, .arg = buf }; int reps = 10; do { debug_printf("!!!!!! NEW ROUND\n"); err = ioat_dma_request_memcpy(dma_ctrl.devices, &setup); assert(err_is_ok(err)); uint32_t i = 10; while(i--) { ioat_dma_device_poll_channels(dma_ctrl.devices); } }while(reps--); } #endif int main(int argc, char *argv[]) { errval_t err; debug_printf("I/O AT DMA driver started\n"); /* * Parsing of cmdline arguments. * * When started by Kaluga, the last element of the cmdline will contain * the basic PCI information of the device. * VENDORID:DEVICEID:BUS:DEV:FUN */ uint32_t vendor_id, device_id; struct pci_addr addr = { .bus = PCI_ADDR_DONT_CARE, .device = PCI_ADDR_DONT_CARE, .device = PCI_ADDR_DONT_CARE }; enum device_type devtype = IOAT_DEVICE_INVAL; if (argc > 1) { uint32_t parsed = sscanf(argv[argc - 1], "%x:%x:%x:%x:%x", &vendor_id, &device_id, &addr.bus, &addr.device, &addr.function); if (parsed != 5) { DEBUGPRINT("WARNING: cmdline parsing failed. Using PCI Address [0,0,0]"); } else { if (vendor_id != 0x8086) { USER_PANIC("unexpected vendor [%x]", vendor_id); } switch ((device_id & 0xFFF0)) { case PCI_DEVICE_IOAT_IVB0: devtype = IOAT_DEVICE_IVB; break; case PCI_DEVICE_IOAT_HSW0: devtype = IOAT_DEVICE_HSW; break; default: USER_PANIC("unexpected device [%x]", device_id) ; break; } DEBUGPRINT("Initializing I/O AT DMA device with PCI address [%u,%u,%u]\n", addr.bus, addr.device, addr.function); } } else { DEBUGPRINT("WARNING: Initializing I/O AT DMA device with unknown PCI address " "[0,0,0]\n"); } err = ioat_device_discovery(addr, devtype, IOAT_DMA_OPERATION); if (err_is_fail(err)) { USER_PANIC_ERR(err, "DMA Device discovery failed"); } #if DMA_BENCH_RUN_BENCHMARK struct ioat_dma_device *dev = ioat_device_get_next(); dma_bench_run_default(dev); #endif #if IOAT_DMA_OPERATION == IOAT_DMA_OPERATION_SERVICE iref_t svc_iref; char svc_name[30]; uint8_t numa_node = (disp_get_core_id() >= 20); snprintf(svc_name, 30, "%s.%u", IOAT_DMA_SERVICE_NAME, numa_node); err = dma_service_init_with_name(svc_name, &dma_svc_cb, NULL, &svc_iref); if (err_is_fail(err)) { USER_PANIC_ERR(err, "Failed to start the DMA service"); } err = dma_manager_register_driver(0, 1ULL << 40, DMA_DEV_TYPE_IOAT, svc_iref); if (err_is_fail(err)) { USER_PANIC_ERR(err, "Failed to register with the DMA manager\n"); } DEBUGPRINT("Driver registered with DMA manager. Serving requests now.\n"); #endif #if IOAT_DMA_OPERATION == IOAT_DMA_OPERATION_LIBRARY #endif uint8_t idle = 0x1; uint32_t idle_counter = 0xFF; while (1) { err = ioat_device_poll(); switch (err_no(err)) { case DMA_ERR_DEVICE_IDLE: idle = idle && 0x1; break; case SYS_ERR_OK: idle = 0; break; default: debug_printf("I/O AT DMA driver terminated: in poll, %s\n", err_getstring(err)); return err; } err = event_dispatch_non_block(get_default_waitset()); switch (err_no(err)) { case SYS_ERR_OK: idle = 0; break; case LIB_ERR_NO_EVENT: idle &= 1; break; default: debug_printf("I/O AT DMA driver terminated in dispatch, %s\n", err_getstring(err)); return err; } if (idle) { idle_counter--; } if (idle_counter == 0) { idle_counter = 0xFF; thread_yield(); } } return 0; }
static errval_t replicate_frame(lvaddr_t addr, struct capref *frame) { errval_t err; #if XOMP_BENCH_WORKER_EN cycles_t repl_timer = bench_tsc(); #endif struct frame_identity id; err = invoke_frame_identify(*frame, &id); if (err_is_fail(err)) { return err; } XWR_DEBUG("Replicating frame: [%016lx]\n", id.base); struct capref replicate; err = frame_alloc(&replicate, (1UL << id.bits), NULL); if (err_is_fail(err)) { return err; } XWR_DEBUG("registering memory with DMA service\n"); #if XOMP_BENCH_WORKER_EN cycles_t register_timer = bench_tsc(); #endif err = dma_register_memory((struct dma_device *) dma_dev, *frame); if (err_is_fail(err)) { return err; } err = dma_register_memory((struct dma_device *) dma_dev, replicate); if (err_is_fail(err)) { return err; } #if XOMP_BENCH_WORKER_EN cycles_t register_timer_end = bench_tsc(); #endif struct dma_req_setup setup = { .done_cb = dma_replication_cb, .cb_arg = NULL, .args = { .memcpy = { .src = id.base, .bytes = (1UL << id.bits) } } }; err = invoke_frame_identify(replicate, &id); if (err_is_fail(err)) { return err; } setup.args.memcpy.dst = id.base; dma_replication_done = 0x0; XWR_DEBUG("DMA request for replication\n"); err = dma_request_memcpy((struct dma_device *)dma_dev, &setup, NULL); if (err_is_fail(err)) { return err; } while (!dma_replication_done) { messages_wait_and_handle_next(); } XWR_DEBUG("Replication done.\n"); *frame = replicate; #if XOMP_BENCH_WORKER_EN cycles_t timer_end = bench_tsc(); debug_printf("%lx replication took %lu cycles, %lu ms\n", worker_id, timer_end - repl_timer, bench_tsc_to_ms(timer_end - repl_timer)); debug_printf("%lx register mem took %lu cycles, %lu ms\n", worker_id, register_timer_end - register_timer, bench_tsc_to_ms(register_timer_end - register_timer)); #endif return SYS_ERR_OK; }
/** * \brief initializes a IOAT DMA device with the giving capability * * \param mmio capability representing the device's MMIO registers * \param dev returns a pointer to the device structure * * \returns SYS_ERR_OK on success * errval on error */ errval_t ioat_dma_device_init(struct capref mmio, struct ioat_dma_device **dev) { errval_t err; struct ioat_dma_device *ioat_device = calloc(1, sizeof(*ioat_device)); if (ioat_device == NULL) { return LIB_ERR_MALLOC_FAIL; } #if DMA_BENCH_ENABLED bench_init(); #endif struct dma_device *dma_dev = &ioat_device->common; struct frame_identity mmio_id; err = invoke_frame_identify(mmio, &mmio_id); if (err_is_fail(err)) { free(ioat_device); return err; } dma_dev->id = device_id++; dma_dev->mmio.paddr = mmio_id.base; dma_dev->mmio.bytes = (1UL << mmio_id.bits); dma_dev->mmio.frame = mmio; IOATDEV_DEBUG("init device with mmio range: {paddr=0x%016lx, size=%u kB}\n", dma_dev->id, mmio_id.base, 1 << mmio_id.bits); err = vspace_map_one_frame_attr((void**) &dma_dev->mmio.vaddr, dma_dev->mmio.bytes, dma_dev->mmio.frame, VREGION_FLAGS_READ_WRITE_NOCACHE, NULL, NULL); if (err_is_fail(err)) { free(ioat_device); return err; } ioat_dma_initialize(&ioat_device->device, NULL, (void *) dma_dev->mmio.vaddr); ioat_device->version = ioat_dma_cbver_rd(&ioat_device->device); IOATDEV_DEBUG("device registers mapped at 0x%016lx. IOAT version: %u.%u\n", dma_dev->id, dma_dev->mmio.vaddr, ioat_dma_cbver_major_extract(ioat_device->version), ioat_dma_cbver_minor_extract(ioat_device->version)); switch (ioat_dma_cbver_major_extract(ioat_device->version)) { case ioat_dma_cbver_1x: err = device_init_ioat_v1(ioat_device); break; case ioat_dma_cbver_2x: err = device_init_ioat_v2(ioat_device); break; case ioat_dma_cbver_3x: err = device_init_ioat_v3(ioat_device); break; default: err = DMA_ERR_DEVICE_UNSUPPORTED; } if (err_is_fail(err)) { vspace_unmap((void*) dma_dev->mmio.vaddr); free(ioat_device); return err; } dma_dev->f.deregister_memory = NULL; dma_dev->f.register_memory = NULL; dma_dev->f.poll = ioat_dma_device_poll_channels; *dev = ioat_device; return err; }
static errval_t elf_allocate(void *state, genvaddr_t base, size_t size, uint32_t flags, void **retbase) { errval_t err; lvaddr_t vaddr; size_t used_size; struct spawninfo *si = state; // Increase size by space wasted on first page due to page-alignment size_t base_offset = BASE_PAGE_OFFSET(base); size += base_offset; base -= base_offset; // Page-align size = ROUND_UP(size, BASE_PAGE_SIZE); cslot_t vspace_slot = si->elfload_slot; // Step 1: Allocate the frames size_t sz = 0; for (lpaddr_t offset = 0; offset < size; offset += sz) { sz = 1UL << log2floor(size - offset); struct capref frame = { .cnode = si->segcn, .slot = si->elfload_slot++, }; err = frame_create(frame, sz, NULL); if (err_is_fail(err)) { return err_push(err, LIB_ERR_FRAME_CREATE); } } cslot_t spawn_vspace_slot = si->elfload_slot; cslot_t new_slot_count = si->elfload_slot - vspace_slot; // Step 2: create copies of the frame capabilities for child vspace for (int copy_idx = 0; copy_idx < new_slot_count; copy_idx++) { struct capref frame = { .cnode = si->segcn, .slot = vspace_slot + copy_idx, }; struct capref spawn_frame = { .cnode = si->segcn, .slot = si->elfload_slot++, }; err = cap_copy(spawn_frame, frame); if (err_is_fail(err)) { debug_printf("cap_copy failed for src_slot = %"PRIuCSLOT ", dest_slot = %"PRIuCSLOT"\n", frame.slot, spawn_frame.slot); return err_push(err, LIB_ERR_CAP_COPY); } } // Step 3: map into own vspace // Get virtual address range to hold the module void *vaddr_range; err = paging_alloc(get_current_paging_state(), &vaddr_range, size); if (err_is_fail(err)) { debug_printf("elf_allocate: paging_alloc failed\n"); return (err); } // map allocated physical memory in virutal memory of parent process vaddr = (lvaddr_t)vaddr_range; used_size = size; while (used_size > 0) { struct capref frame = { .cnode = si->segcn, .slot = vspace_slot++, }; // find out the size of the frame struct frame_identity id; err = invoke_frame_identify(frame, &id); assert(err_is_ok(err)); size_t slot_size = (1UL << id.bits); // map frame to provide physical memory backing err = paging_map_fixed_attr(get_current_paging_state(), vaddr, frame, slot_size, VREGION_FLAGS_READ_WRITE); if (err_is_fail(err)) { debug_printf("elf_allocate: paging_map_fixed_attr failed\n"); return err; } used_size -= slot_size; vaddr += slot_size; } // end while: // Step 3: map into new process struct paging_state *cp = si->vspace; // map allocated physical memory in virutal memory of child process vaddr = (lvaddr_t)base; used_size = size; while (used_size > 0) { struct capref frame = { .cnode = si->segcn, .slot = spawn_vspace_slot++, }; // find out the size of the frame struct frame_identity id; err = invoke_frame_identify(frame, &id); assert(err_is_ok(err)); size_t slot_size = (1UL << id.bits); // map frame to provide physical memory backing err = paging_map_fixed_attr(cp, vaddr, frame, slot_size, elf_to_vregion_flags(flags)); if (err_is_fail(err)) { debug_printf("elf_allocate: paging_map_fixed_attr failed\n"); return err; } used_size -= slot_size; vaddr += slot_size; } // end while: *retbase = (void*) vaddr_range + base_offset; return SYS_ERR_OK; } // end function: elf_allocate /** * \brief Load the elf image */ errval_t spawn_arch_load(struct spawninfo *si, lvaddr_t binary, size_t binary_size, genvaddr_t *entry, void** arch_info) { errval_t err; // Reset the elfloader_slot si->elfload_slot = 0; struct capref cnode_cap = { .cnode = si->rootcn, .slot = ROOTCN_SLOT_SEGCN, }; err = cnode_create_raw(cnode_cap, &si->segcn, DEFAULT_CNODE_SLOTS, NULL); if (err_is_fail(err)) { return err_push(err, SPAWN_ERR_CREATE_SEGCN); } // TLS is NYI si->tls_init_base = 0; si->tls_init_len = si->tls_total_len = 0; //debug_printf("spawn_arch_load: about to load elf %p\n", elf_allocate); // Load the binary err = elf_load(EM_HOST, elf_allocate, si, binary, binary_size, entry); if (err_is_fail(err)) { return err; } //debug_printf("hello here\n"); struct Elf32_Shdr* got_shdr = elf32_find_section_header_name(binary, binary_size, ".got"); if (got_shdr) { *arch_info = (void*)got_shdr->sh_addr; } else { return SPAWN_ERR_LOAD; } return SYS_ERR_OK; } void spawn_arch_set_registers(void *arch_load_info, dispatcher_handle_t handle, arch_registers_state_t *enabled_area, arch_registers_state_t *disabled_area) { assert(arch_load_info != NULL); uintptr_t got_base = (uintptr_t)arch_load_info; struct dispatcher_shared_arm* disp_arm = get_dispatcher_shared_arm(handle); disp_arm->got_base = got_base; enabled_area->regs[REG_OFFSET(PIC_REGISTER)] = got_base; enabled_area->named.cpsr = CPSR_F_MASK | ARM_MODE_USR; disabled_area->regs[REG_OFFSET(PIC_REGISTER)] = got_base; disabled_area->named.cpsr = CPSR_F_MASK | ARM_MODE_USR; }
/** * \brief A monitor receives request to setup a connection * with another newly booted monitor from a third monitor */ static void bind_monitor_request_scc(struct intermon_binding *b, coreid_t core_id, intermon_caprep_t caprep, chanid_t chan_id, coreid_t from_core_id) { struct intermon_ump_ipi_binding *umpb = NULL; errval_t err; /* Create the cap */ struct capability cap_raw; caprep_to_capability(&caprep, &cap_raw); if (cap_raw.type != ObjType_Frame) { err = MON_ERR_WRONG_CAP_TYPE; goto error; } struct capref frame; err = slot_alloc(&frame); if (err_is_fail(err)) { goto error; } ram_set_affinity(cap_raw.u.frame.base, cap_raw.u.frame.base + ((genpaddr_t)1 << cap_raw.u.frame.bits)); err = frame_alloc(&frame, ((genpaddr_t)1 << cap_raw.u.frame.bits), NULL); ram_set_affinity(0,0); /* err = monitor_cap_create(frame, &cap_raw, core_id); */ if (err_is_fail(err)) { goto error; } struct frame_identity frameid = { .base = 0, .bits = 0 }; err = invoke_frame_identify(frame, &frameid); assert(err == SYS_ERR_OK); printf("bind_monitor_request: URPC physical frame at 0x%llx\n", frameid.base); /* Setup the connection */ void *buf; err = vspace_map_one_frame_attr(&buf, MON_URPC_SIZE, frame, VREGION_FLAGS_READ_WRITE_NOCACHE, NULL, NULL); if (err_is_fail(err)) { err = err_push(err, LIB_ERR_VSPACE_MAP); goto error; } // Create remote notify cap struct capref notify_cap; err = notification_create_cap(chan_id, core_id, ¬ify_cap); assert(err == SYS_ERR_OK); // Allocate my own notification caps struct capref ep, my_notify_cap; struct lmp_endpoint *iep; int chanid; err = endpoint_create(LMP_RECV_LENGTH, &ep, &iep); assert(err_is_ok(err)); err = notification_allocate(ep, &chanid); assert(err == SYS_ERR_OK); err = notification_create_cap(chanid, my_core_id, &my_notify_cap); assert(err == SYS_ERR_OK); // setup our side of the binding umpb = malloc(sizeof(struct intermon_ump_ipi_binding)); assert(umpb != NULL); err = intermon_ump_ipi_init(umpb, get_default_waitset(), buf + MON_URPC_CHANNEL_LEN, MON_URPC_CHANNEL_LEN, buf, MON_URPC_CHANNEL_LEN, notify_cap, my_notify_cap, ep, iep); assert(err_is_ok(err)); // Identify UMP frame for tracing struct frame_identity umpid; err = invoke_frame_identify(frame, &umpid); assert(err_is_ok(err)); umpb->ump_state.chan.recvid = (uintptr_t)umpid.base; umpb->ump_state.chan.sendid = (uintptr_t)(umpid.base + MON_URPC_CHANNEL_LEN); // connect it to our request handlers err = intermon_init(&umpb->b, core_id); assert(err_is_ok(err)); /* Send reply */ reply: assert(umpb != NULL); bind_monitor_reply_scc_cont(&umpb->b, err, chanid); return; error: assert(!"Argh"); // FIXME: cleanup! goto reply; } /** * \brief The monitor that proxied the request for one monitor to * setup a connection with another monitor gets the reply */ static void bind_monitor_reply_scc(struct intermon_binding *binding, errval_t err, chanid_t chan_id, coreid_t core_id) { struct intermon_ump_ipi_binding *b = (struct intermon_ump_ipi_binding *)binding; // Create notify cap to that core struct capref notify_cap; err = notification_create_cap(chan_id, core_id, ¬ify_cap); assert(err == SYS_ERR_OK); // And assign it to the binding err = ipi_notify_set(&b->ipi_notify, notify_cap); assert(err_is_ok(err)); if (err_is_fail(err)) { // XXX DEBUG_ERR(err, "Got error in bind monitor reply"); } } /******* stack-ripped bind_monitor_proxy_scc *******/ static void bind_monitor_request_scc_handler(struct intermon_binding *b, struct intermon_msg_queue_elem *e); struct bind_monitor_request_scc_state { struct intermon_msg_queue_elem elem; struct intermon_bind_monitor_request_scc__args args; }; static void bind_monitor_request_scc_cont(struct intermon_binding *dst_binding, coreid_t src_core_id, intermon_caprep_t caprep, chanid_t chan_id, coreid_t core_id) { errval_t err; err = dst_binding->tx_vtbl. bind_monitor_request_scc(dst_binding, NOP_CONT, src_core_id, caprep, chan_id, core_id); if (err_is_fail(err)) { if(err_no(err) == FLOUNDER_ERR_TX_BUSY) { struct bind_monitor_request_scc_state *me = malloc(sizeof(struct bind_monitor_request_scc_state)); assert(me != NULL); struct intermon_state *ist = dst_binding->st; assert(ist != NULL); me->args.core_id = src_core_id; me->args.cap = caprep; me->args.chan_id = chan_id; me->args.from_core_id = core_id; me->elem.cont = bind_monitor_request_scc_handler; err = intermon_enqueue_send(dst_binding, &ist->queue, get_default_waitset(), &me->elem.queue); assert(err_is_ok(err)); return; } DEBUG_ERR(err, "forwarding bind request failed"); } }
int map_unmap(void) { errval_t err; struct capref mem; DEBUG_MAP_UNMAP("ram_alloc\n"); err = ram_alloc(&mem, BASE_PAGE_BITS); if (err_is_fail(err)) { printf("ram_alloc: %s (%"PRIuERRV")\n", err_getstring(err), err); return 1; } struct capref frame; DEBUG_MAP_UNMAP("retype\n"); err = slot_alloc(&frame); if (err_is_fail(err)) { printf("slot_alloc: %s (%"PRIuERRV")\n", err_getstring(err), err); return 1; } err = cap_retype(frame, mem, ObjType_Frame, BASE_PAGE_BITS); if (err_is_fail(err)) { printf("cap_retype: %s (%"PRIuERRV")\n", err_getstring(err), err); return 1; } DEBUG_MAP_UNMAP("delete ram cap\n"); err = cap_destroy(mem); if (err_is_fail(err)) { printf("cap_delete(mem): %s (%"PRIuERRV")\n", err_getstring(err), err); return 1; } struct frame_identity fi; err = invoke_frame_identify(frame, &fi); if (err_is_fail(err)) { printf("frame_identify: %s (%"PRIuERRV")\n", err_getstring(err), err); return 1; } DEBUG_MAP_UNMAP("frame: base = 0x%"PRIxGENPADDR", bits = %d\n", fi.base, fi.bits); #ifdef NKMTEST_DEBUG_MAP_UNMAP dump_pmap(get_current_pmap()); #endif struct vregion *vr; struct memobj *memobj; void *vaddr; DEBUG_MAP_UNMAP("map\n"); err = vspace_map_one_frame(&vaddr, BASE_PAGE_SIZE, frame, &memobj, &vr); if (err_is_fail(err)) { printf("vspace_map_one_frame: %s (%"PRIuERRV")\n", err_getstring(err), err); } char *memory = vaddr; DEBUG_MAP_UNMAP("vaddr = %p\n", vaddr); #ifdef NKMTEST_DEBUG_MAP_UNMAP dump_pmap(get_current_pmap()); #endif DEBUG_MAP_UNMAP("write 1\n"); int i; for (i = 0; i < BASE_PAGE_SIZE; i++) { memory[i] = i % INT8_MAX; } DEBUG_MAP_UNMAP("verify 1\n"); for (i = 0; i < BASE_PAGE_SIZE; i++) { assert(memory[i] == i % INT8_MAX); } DEBUG_MAP_UNMAP("delete frame cap\n"); err = cap_destroy(frame); if (err_is_fail(err)) { printf("cap_delete(frame): %s (%"PRIuERRV")\n", err_getstring(err), err); return 1; } #ifdef NKMTEST_DEBUG_MAP_UNMAP // no mapping should remain here dump_pmap(get_current_pmap()); err = debug_dump_hw_ptables(); if (err_is_fail(err)) { printf("kernel dump ptables: %s (%"PRIuERRV")\n", err_getstring(err), err); return 1; } #endif printf("%s: done\n", __FUNCTION__); return 0; }
static void gw_req_memory_call_rx(struct xomp_binding *b, uint64_t addr, uint8_t type) { XWI_DEBUG("gw_req_memory_call_rx: addr:%lx, tyep: %u\n", addr, type); #if XOMP_BENCH_WORKER_EN cycles_t mem_timer = bench_tsc(); #endif struct txq_msg_st *msg_st = txq_msg_st_alloc(&txq); assert(msg_st != NULL); struct capref frame; if (type == XOMP_FRAME_TYPE_REPL_RW) { type = XOMP_FRAME_TYPE_SHARED_RW; } assert(!(worker_id & XOMP_WID_GATEWAY_FLAG)); msg_st->send = gw_req_memory_response_tx; msg_st->cleanup = NULL; XWR_DEBUG("Requesting frame from gateway: [%016lx]\n", usrdata); msg_st->err = xomp_gateway_get_memory(addr, &frame); if (err_is_fail(msg_st->err)) { txq_send(msg_st); return; } vregion_flags_t map_flags; switch ((xomp_frame_type_t) type) { case XOMP_FRAME_TYPE_MSG: map_flags = VREGION_FLAGS_READ_WRITE; break; case XOMP_FRAME_TYPE_SHARED_RW: case XOMP_FRAME_TYPE_REPL_RW: map_flags = VREGION_FLAGS_READ_WRITE; break; case XOMP_FRAME_TYPE_SHARED_RO: map_flags = VREGION_FLAGS_READ; break; default: USER_PANIC("unknown type: %u", type) break; } struct frame_identity id; msg_st->err = invoke_frame_identify(frame, &id); if (err_is_fail(msg_st->err)) { txq_send(msg_st); return; } if (addr) { msg_st->err = vspace_map_one_frame_fixed_attr(addr, (1UL << id.bits), frame, map_flags, NULL, NULL); } else { void *map_addr; msg_st->err = vspace_map_one_frame_attr(&map_addr, (1UL << id.bits), frame, map_flags, NULL, NULL); } #if XOMP_BENCH_WORKER_EN mem_timer = bench_tsc() - mem_timer; debug_printf("%lx mem request %016lx took %lu cycles, %lu ms\n", worker_id, addr, mem_timer, bench_tsc_to_ms(mem_timer)); #endif txq_send(msg_st); }
static void monitor_bind_ump_client_request(struct monitor_binding *mb, iref_t iref, uintptr_t domain_id, struct capref frame, size_t channel_length_in, size_t channel_length_out, struct capref notify) { uint8_t core_id; uintptr_t conn_id = 0; errval_t err; struct remote_conn_state *conn = NULL; // Get the core id err = iref_get_core_id(iref, &core_id); if (err_is_fail(err)) { debug_err(__FILE__, __func__, __LINE__, err, "iref_get_core_id failed"); monitor_bind_ump_client_request_error(mb, frame, conn_id, domain_id, err); return; } if (core_id == my_core_id) { USER_PANIC("Same-core UMP binding NYI"); } /* Identify frame */ struct frame_identity frameid; err = invoke_frame_identify(frame, &frameid); if (err_is_fail(err)) { debug_err(__FILE__, __func__, __LINE__, err, "frame_identify failed"); monitor_bind_ump_client_request_error(mb, frame, conn_id, domain_id, err); return; } // Identify notify cap struct capability capability; err = monitor_cap_identify(notify, &capability); if (err_is_fail(err)) { USER_PANIC_ERR(err, "monitor_cap_identify failed, ignored"); return; } assert(capability.type == ObjType_Notify_RCK || capability.type == ObjType_Notify_IPI || capability.type == ObjType_Null); /* assert(capability.u.notify.coreid == my_core_id); */ /* Forward request to the corresponding monitor */ // Create local state err = remote_conn_alloc(&conn, &conn_id, REMOTE_CONN_UMP); if (err_is_fail(err)) { debug_err(__FILE__, __func__, __LINE__, err, "remote_conn_alloc failed"); monitor_bind_ump_client_request_error(mb, frame, conn_id, domain_id, err); return; } // Track data conn->domain_id = domain_id; conn->domain_binding = mb; conn->x.ump.frame = frame; conn->core_id = core_id; // Get connection to the monitor to forward request to struct intermon_binding *intermon_binding; err = intermon_binding_get(core_id, &intermon_binding); if (err_is_fail(err)) { debug_err(__FILE__, __func__, __LINE__, err, "intermon_binding_get failed"); monitor_bind_ump_client_request_error(mb, frame, conn_id, domain_id, err); return; } bind_ump_request_cont(intermon_binding, iref, conn_id, channel_length_in, channel_length_out, frameid, capability, mb, frame, domain_id); }
/** * \brief initializes the XOMP worker library * * \param wid Xomp worker id * * \returns SYS_ERR_OK on success * errval on failure */ errval_t xomp_worker_init(xomp_wid_t wid) { errval_t err; worker_id = wid; XWI_DEBUG("initializing worker {%016lx} iref:%u\n", worker_id, svc_iref); #if XOMP_BENCH_WORKER_EN bench_init(); #endif struct capref frame = { .cnode = cnode_root, .slot = ROOTCN_SLOT_ARGCN }; struct frame_identity id; err = invoke_frame_identify(frame, &id); if (err_is_fail(err)) { return err_push(err, XOMP_ERR_INVALID_MSG_FRAME); } size_t frame_size = 0; if (svc_iref) { frame_size = XOMP_TLS_SIZE; } else { frame_size = XOMP_FRAME_SIZE; err = spawn_symval_cache_init(0); if (err_is_fail(err)) { return err; } } if ((1UL << id.bits) < XOMP_TLS_SIZE) { return XOMP_ERR_INVALID_MSG_FRAME; } msgframe = frame; err = vspace_map_one_frame(&msgbuf, frame_size, frame, NULL, NULL); if (err_is_fail(err)) { err_push(err, XOMP_ERR_WORKER_INIT_FAILED); } if (svc_iref) { tls = msgbuf; } else { tls = ((uint8_t *) msgbuf) + XOMP_MSG_FRAME_SIZE; } XWI_DEBUG("messaging frame mapped: [%016lx] @ [%016lx]\n", id.base, (lvaddr_t )msgbuf); struct bomp_thread_local_data *tlsinfo = malloc(sizeof(*tlsinfo)); tlsinfo->thr = thread_self(); tlsinfo->work = (struct bomp_work *) tls; tlsinfo->work->data = tlsinfo->work + 1; g_bomp_state->backend.set_tls(tlsinfo); #ifdef __k1om__ if (worker_id & XOMP_WID_GATEWAY_FLAG) { err = xomp_gateway_init(); } else { if (!svc_iref) { err = xomp_gateway_bind_svc(); } else { err = SYS_ERR_OK; } } if (err_is_fail(err)) { return err; } #endif #ifdef __k1om__ if (!svc_iref) { err = xeon_phi_client_init(disp_xeon_phi_id()); if (err_is_fail(err)) { err_push(err, XOMP_ERR_WORKER_INIT_FAILED); } xeon_phi_client_set_callbacks(&callbacks); } #endif struct waitset *ws = get_default_waitset(); // XXX: disabling DMA on the host as there is no replication used at this moment #if XOMP_WORKER_ENABLE_DMA && defined(__k1om__) /* XXX: use lib numa */ #ifndef __k1om__ uint8_t numanode = 0; if (disp_get_core_id() > 20) { numanode = 1; } err = dma_manager_wait_for_driver(dma_device_type, numanode); if (err_is_fail(err)) { USER_PANIC_ERR(err, "could not wait for the DMA driver"); } #endif char svc_name[30]; #ifdef __k1om__ snprintf(svc_name, 30, "%s", XEON_PHI_DMA_SERVICE_NAME); #else snprintf(svc_name, 30, "%s.%u", IOAT_DMA_SERVICE_NAME, numanode); #endif struct dma_client_info dma_info = { .type = DMA_CLIENT_INFO_TYPE_NAME, .device_type = dma_device_type, .args.name = svc_name }; err = dma_client_device_init(&dma_info, &dma_dev); if (err_is_fail(err)) { USER_PANIC_ERR(err, "DMA device initialization"); } #endif if (svc_iref) { err = xomp_bind(svc_iref, master_bind_cb, NULL, ws, IDC_EXPORT_FLAGS_DEFAULT); } else { struct xomp_frameinfo fi = { .sendbase = id.base, .inbuf = ((uint8_t *) msgbuf) + XOMP_MSG_CHAN_SIZE, .inbufsize = XOMP_MSG_CHAN_SIZE, .outbuf = ((uint8_t *) msgbuf), .outbufsize = XOMP_MSG_CHAN_SIZE }; err = xomp_connect(&fi, master_bind_cb, NULL, ws, IDC_EXPORT_FLAGS_DEFAULT); } if (err_is_fail(err)) { /* TODO: Clean up */ return err_push(err, XOMP_ERR_WORKER_INIT_FAILED); } XWI_DEBUG("Waiting until bound to master...\n"); while (!is_bound) { messages_wait_and_handle_next(); } if (xbinding == NULL) { return XOMP_ERR_WORKER_INIT_FAILED; } return SYS_ERR_OK; }
static void span_domain_request(struct monitor_binding *mb, uintptr_t domain_id, uint8_t core_id, struct capref vroot, struct capref disp) { errval_t err, err2; trace_event(TRACE_SUBSYS_MONITOR, TRACE_EVENT_MONITOR_SPAN0, core_id); struct span_state *state; uintptr_t state_id; err = span_state_alloc(&state, &state_id); if (err_is_fail(err)) { err_push(err, MON_ERR_SPAN_STATE_ALLOC); goto reply; } state->core_id = core_id; state->vroot = vroot; state->mb = mb; state->domain_id = domain_id; trace_event(TRACE_SUBSYS_MONITOR, TRACE_EVENT_MONITOR_SPAN1, core_id); /* Look up the destination monitor */ struct intermon_binding *ib; err = intermon_binding_get(core_id, &ib); if (err_is_fail(err)) { goto reply; } /* Idenfity vroot */ struct capability vroot_cap; err = monitor_cap_identify(vroot, &vroot_cap); if (err_is_fail(err)) { err_push(err, MON_ERR_CAP_IDENTIFY); goto reply; } if (vroot_cap.type != ObjType_VNode_x86_64_pml4) { /* Check type */ err = MON_ERR_WRONG_CAP_TYPE; goto reply; } /* Identify the dispatcher frame */ struct frame_identity frameid; err = invoke_frame_identify(disp, &frameid); if (err_is_fail(err)) { err_push(err, LIB_ERR_FRAME_IDENTIFY); goto reply; } err = monitor_remote_relations(disp, RRELS_COPY_BIT, RRELS_COPY_BIT, NULL); if (err_is_fail(err)) { USER_PANIC_ERR(err, "monitor_remote_relations failed"); return; } err = monitor_remote_relations(vroot, RRELS_COPY_BIT, RRELS_COPY_BIT, NULL); if (err_is_fail(err)) { USER_PANIC_ERR(err, "monitor_remote_relations failed"); return; } /* Send msg to destination monitor */ err = ib->tx_vtbl.span_domain_request(ib, NOP_CONT, state_id, vroot_cap.u.vnode_x86_64_pml4.base, frameid.base, frameid.bits); if (err_is_fail(err)) { err_push(err, MON_ERR_SEND_REMOTE_MSG); goto reply; } goto cleanup; reply: err2 = mb->tx_vtbl.span_domain_reply(mb, NOP_CONT, err, domain_id); if (err_is_fail(err2)) { // XXX: Cleanup? USER_PANIC_ERR(err2, "Failed to reply to the user domain"); } if(state_id != 0) { err2 = span_state_free(state_id); if (err_is_fail(err2)) { USER_PANIC_ERR(err2, "Failed to free span state"); } } cleanup: err2 = cap_destroy(vroot); if (err_is_fail(err2)) { USER_PANIC_ERR(err2, "Failed to destroy span_vroot cap"); } err2 = cap_destroy(disp); if (err_is_fail(err2)) { USER_PANIC_ERR(err2, "Failed to destroy disp cap"); } }
/* * ---------------------------------------------------------------------------- * Xeon Phi Channel callbacks * ---------------------------------------------------------------------------- */ static errval_t msg_open_cb(xphi_dom_id_t domain, uint64_t usrdata, struct capref frame, uint8_t type) { errval_t err; uint32_t map_flags = 0x0; lvaddr_t addr = 0x0; struct frame_identity id; err = invoke_frame_identify(frame, &id); if (err_is_fail(err)) { return err; } XWI_DEBUG("msg_open_cb: from domid:%lx, usrdata:%lx, frame:%lx\n", domain, usrdata, id.base); switch ((xomp_frame_type_t) type) { case XOMP_FRAME_TYPE_MSG: map_flags = VREGION_FLAGS_READ_WRITE; break; case XOMP_FRAME_TYPE_SHARED_RW: addr = (lvaddr_t) usrdata; map_flags = VREGION_FLAGS_READ_WRITE; break; case XOMP_FRAME_TYPE_SHARED_RO: map_flags = VREGION_FLAGS_READ; break; case XOMP_FRAME_TYPE_REPL_RW: map_flags = VREGION_FLAGS_READ_WRITE; #if XOMP_WORKER_ENABLE_DMA addr = (lvaddr_t) usrdata; err = replicate_frame(addr, &frame); if (err_is_fail(err)) { return err; } err = invoke_frame_identify(frame, &id); #else struct capref replicate; err = frame_alloc(&replicate, (1UL << id.bits), NULL); if (err_is_fail(err)) { USER_PANIC_ERR(err, "failed to allocate replicate frame\n"); return err; } err = vspace_map_one_frame_fixed_attr((lvaddr_t) usrdata, (1UL << id.bits), replicate, map_flags, NULL, NULL); if (err_is_fail(err)) { return err; } err = invoke_frame_identify(replicate, &id); #endif if (err_is_fail(err)) { return err; } break; default: USER_PANIC("unknown type: %u", type) break; } if (addr) { if (worker_id & XOMP_WID_GATEWAY_FLAG) { XWR_DEBUG("registering memory with gateway: [%016lx]\n", addr); err = xomp_gateway_mem_insert(frame, addr); if (err_is_fail(err)) { /* todo: cleanup */ return err; } } err = vspace_map_one_frame_fixed_attr(addr, (1UL << id.bits), frame, map_flags, NULL, NULL); } else { err = vspace_map_one_frame_attr((void **) &addr, (1UL << id.bits), frame, map_flags, NULL, NULL); } if (err_is_fail(err)) { return err; } #if !XOMP_WORKER_ENABLE_DMA if ((xomp_frame_type_t) type == XOMP_FRAME_TYPE_REPL_RW) { memcpy((void *)usrdata, (void *)addr, (1UL << id.bits)); } #endif XWI_DEBUG("msg_open_cb: frame [%016lx] mapped @ [%016lx, %016lx]\n", id.base, addr, addr + (1UL << id.bits)); if ((xomp_frame_type_t) type == XOMP_FRAME_TYPE_MSG) { USER_PANIC("NYI: initializing messaging"); } return SYS_ERR_OK; }