/* mark the physical pages backing a range of virtual as in use.
* allocate the physical pages and throw them away */
static void mark_pages_in_use(vaddr_t va, size_t len)
{
LTRACEF("va 0x%lx, len 0x%zx\n", va, len);
struct list_node list;
list_initialize(&list);
/* make sure we are inclusive of all of the pages in the address range */
len = PAGE_ALIGN(len + (va & (PAGE_SIZE - 1)));
va = ROUNDDOWN(va, PAGE_SIZE);
LTRACEF("aligned va 0x%lx, len 0x%zx\n", va, len);
for (size_t offset = 0; offset < len; offset += PAGE_SIZE) {
uint flags;
paddr_t pa;
status_t err = arch_mmu_query(va + offset, &pa, &flags);
if (err >= 0) {
//LTRACEF("va 0x%x, pa 0x%x, flags 0x%x, err %d\n", va + offset, pa, flags, err);
/* alloate the range, throw the results away */
pmm_alloc_range(pa, 1, &list);
} else {
panic("Could not find pa for va 0x%lx\n", va);
}
}
}
paddr_t khook_arch_virt2phys(khook_t* khook, kvaddr_t va) {
paddr_t pa;
if(arch_mmu_query(khook, va, &pa, NULL, NULL))
return 0;
return pa;
}
paddr_t kvaddr_to_paddr(void *ptr)
{
status_t rc;
paddr_t pa;
rc = arch_mmu_query((vaddr_t)ptr, &pa, NULL);
if (rc)
return (paddr_t) NULL;
return pa;
}
static int mem_test(int argc, const cmd_args *argv)
{
if (argc < 2) {
printf("not enough arguments\n");
usage:
printf("usage: %s <length>\n", argv[0].str);
return -1;
}
void *ptr;
size_t len = argv[1].u;
#if WITH_KERNEL_VMM
/* rounding up len to the next page */
len = PAGE_ALIGN(len);
if (len == 0) {
printf("invalid length\n");
goto usage;
}
/* allocate a region to test in */
status_t err = vmm_alloc_contiguous(vmm_get_kernel_aspace(), "memtest", len, &ptr, 0, 0, ARCH_MMU_FLAG_UNCACHED);
if (err < 0) {
printf("error %d allocating test region\n", err);
return -1;
}
paddr_t pa;
arch_mmu_query((vaddr_t)ptr, &pa, 0);
printf("physical address 0x%lx\n", pa);
#else
/* allocate from the heap */
ptr = malloc(len);
if (!ptr ) {
printf("error allocating test area from heap\n");
return -1;
}
#endif
printf("got buffer at %p of length 0x%lx\n", ptr, len);
/* run the tests */
do_mem_tests(ptr, len);
#if WITH_KERNEL_VMM
// XXX free memory region here
printf("NOTE: leaked memory\n");
#else
free(ptr);
#endif
return 0;
}
void lkboot_dcc_init(void)
{
paddr_t pa;
__UNUSED status_t err;
buffer_desc.version = PDCC_VERSION;
err = arch_mmu_query((vaddr_t)htod_buffer, &pa, NULL);
DEBUG_ASSERT(err == NO_ERROR);
buffer_desc.htod_buffer_phys = pa;
buffer_desc.htod_buffer_len = DCC_BUFLEN;
err = arch_mmu_query((vaddr_t)dtoh_buffer, &pa, NULL);
DEBUG_ASSERT(err == NO_ERROR);
buffer_desc.dtoh_buffer_phys = pa;
buffer_desc.dtoh_buffer_len = DCC_BUFLEN;
err = arch_mmu_query((vaddr_t)&buffer_desc, &buffer_desc_phys, NULL);
DEBUG_ASSERT(err == NO_ERROR);
arch_clean_cache_range((vaddr_t)&buffer_desc, sizeof(buffer_desc));
}
static int cmd_vm(int argc, const cmd_args *argv)
{
if (argc < 2) {
notenoughargs:
printf("not enough arguments\n");
usage:
printf("usage:\n");
printf("%s phys2virt <address>\n", argv[0].str);
printf("%s virt2phys <address>\n", argv[0].str);
printf("%s map <phys> <virt> <count> <flags>\n", argv[0].str);
printf("%s unmap <virt> <count>\n", argv[0].str);
return ERR_GENERIC;
}
if (!strcmp(argv[1].str, "phys2virt")) {
if (argc < 3) goto notenoughargs;
void *ptr = paddr_to_kvaddr(argv[2].u);
printf("paddr_to_kvaddr returns %p\n", ptr);
} else if (!strcmp(argv[1].str, "virt2phys")) {
if (argc < 3) goto notenoughargs;
paddr_t pa;
uint flags;
status_t err = arch_mmu_query(argv[2].u, &pa, &flags);
printf("arch_mmu_query returns %d\n", err);
if (err >= 0) {
printf("\tpa 0x%lx, flags 0x%x\n", pa, flags);
}
} else if (!strcmp(argv[1].str, "map")) {
if (argc < 6) goto notenoughargs;
int err = arch_mmu_map(argv[3].u, argv[2].u, argv[4].u, argv[5].u);
printf("arch_mmu_map returns %d\n", err);
} else if (!strcmp(argv[1].str, "unmap")) {
if (argc < 4) goto notenoughargs;
int err = arch_mmu_unmap(argv[2].u, argv[3].u);
printf("arch_mmu_unmap returns %d\n", err);
} else {
printf("unknown command\n");
goto usage;
}
return NO_ERROR;
}
static status_t attach_backing(struct virtio_gpu_dev *gdev, uint32_t resource_id, void *ptr, size_t buf_len)
{
status_t err;
LTRACEF("gdev %p, resource_id %u, ptr %p, buf_len %zu\n", gdev, resource_id, ptr, buf_len);
DEBUG_ASSERT(gdev);
DEBUG_ASSERT(ptr);
/* grab a lock to keep this single message at a time */
mutex_acquire(&gdev->lock);
/* construct the request */
struct {
struct virtio_gpu_resource_attach_backing req;
struct virtio_gpu_mem_entry mem;
} req;
memset(&req, 0, sizeof(req));
req.req.hdr.type = VIRTIO_GPU_CMD_RESOURCE_ATTACH_BACKING;
req.req.resource_id = resource_id;
req.req.nr_entries = 1;
paddr_t pa;
arch_mmu_query((vaddr_t)ptr, &pa, NULL);
req.mem.addr = pa;
req.mem.length = buf_len;
/* send the command and get a response */
struct virtio_gpu_ctrl_hdr *res;
err = send_command_response(gdev, &req, sizeof(req), (void **)&res, sizeof(*res));
DEBUG_ASSERT(err == NO_ERROR);
/* see if we got a valid response */
LTRACEF("response type 0x%x\n", res->type);
err = (res->type == VIRTIO_GPU_RESP_OK_NODATA) ? NO_ERROR : ERR_NO_MEMORY;
/* release the lock */
mutex_release(&gdev->lock);
return err;
}
static int alloc_page_table(paddr_t *paddrp, uint page_size_shift)
{
int ret;
int count;
size_t size = 1U << page_size_shift;
void *vaddr;
if (size >= PAGE_SIZE) {
count = size / PAGE_SIZE;
ret = pmm_alloc_contiguous(count, page_size_shift, paddrp, NULL, PMM_ARENA_FLAG_ANY);
if (ret != count)
return ERR_NO_MEMORY;
} else {
vaddr = heap_alloc(size, size);
if (!vaddr)
return ERR_NO_MEMORY;
ret = arch_mmu_query((vaddr_t)vaddr, paddrp, NULL);
if (ret) {
heap_free(vaddr);
return ret;
}
}
return 0;
}
status_t virtio_block_init(struct virtio_device *dev, uint32_t host_features)
{
LTRACEF("dev %p, host_features 0x%x\n", dev, host_features);
/* allocate a new block device */
struct virtio_block_dev *bdev = malloc(sizeof(struct virtio_block_dev));
if (!bdev)
return ERR_NO_MEMORY;
mutex_init(&bdev->lock);
event_init(&bdev->io_event, false, EVENT_FLAG_AUTOUNSIGNAL);
bdev->dev = dev;
dev->priv = bdev;
bdev->blk_req = memalign(sizeof(struct virtio_blk_req), sizeof(struct virtio_blk_req));
#if WITH_KERNEL_VM
arch_mmu_query((vaddr_t)bdev->blk_req, &bdev->blk_req_phys, NULL);
#else
bdev->blk_freq_phys = (uint64_t)(uintptr_t)bdev->blk_req;
#endif
LTRACEF("blk_req structure at %p (0x%lx phys)\n", bdev->blk_req, bdev->blk_req_phys);
#if WITH_KERNEL_VM
arch_mmu_query((vaddr_t)&bdev->blk_response, &bdev->blk_response_phys, NULL);
#else
bdev->blk_response_phys = (uint64_t)(uintptr_t)&bdev->blk_response;
#endif
/* make sure the device is reset */
virtio_reset_device(dev);
volatile struct virtio_blk_config *config = (struct virtio_blk_config *)dev->config_ptr;
LTRACEF("capacity 0x%llx\n", config->capacity);
LTRACEF("size_max 0x%x\n", config->size_max);
LTRACEF("seg_max 0x%x\n", config->seg_max);
LTRACEF("blk_size 0x%x\n", config->blk_size);
/* ack and set the driver status bit */
virtio_status_acknowledge_driver(dev);
// XXX check features bits and ack/nak them
/* allocate a virtio ring */
virtio_alloc_ring(dev, 0, 256);
/* set our irq handler */
dev->irq_driver_callback = &virtio_block_irq_driver_callback;
/* set DRIVER_OK */
virtio_status_driver_ok(dev);
/* construct the block device */
static uint8_t found_index = 0;
char buf[16];
snprintf(buf, sizeof(buf), "virtio%u", found_index++);
bio_initialize_bdev(&bdev->bdev, buf,
config->blk_size, config->capacity,
0, NULL);
/* override our block device hooks */
bdev->bdev.read_block = &virtio_bdev_read_block;
bdev->bdev.write_block = &virtio_bdev_write_block;
bio_register_device(&bdev->bdev);
printf("found virtio block device of size %lld\n", config->capacity * config->blk_size);
return NO_ERROR;
}
ssize_t virtio_block_read_write(struct virtio_device *dev, void *buf, off_t offset, size_t len, bool write)
{
struct virtio_block_dev *bdev = (struct virtio_block_dev *)dev->priv;
uint16_t i;
struct vring_desc *desc;
paddr_t pa;
vaddr_t va = (vaddr_t)buf;
LTRACEF("dev %p, buf %p, offset 0x%llx, len %zu\n", dev, buf, offset, len);
mutex_acquire(&bdev->lock);
/* set up the request */
bdev->blk_req->type = write ? VIRTIO_BLK_T_OUT : VIRTIO_BLK_T_IN;
bdev->blk_req->ioprio = 0;
bdev->blk_req->sector = offset / 512;
LTRACEF("blk_req type %u ioprio %u sector %llu\n",
bdev->blk_req->type, bdev->blk_req->ioprio, bdev->blk_req->sector);
/* put together a transfer */
desc = virtio_alloc_desc_chain(dev, 0, 3, &i);
LTRACEF("after alloc chain desc %p, i %u\n", desc, i);
// XXX not cache safe.
// At the moment only tested on arm qemu, which doesn't emulate cache.
/* set up the descriptor pointing to the head */
desc->addr = bdev->blk_req_phys;
desc->len = sizeof(struct virtio_blk_req);
desc->flags |= VRING_DESC_F_NEXT;
/* set up the descriptor pointing to the buffer */
desc = virtio_desc_index_to_desc(dev, 0, desc->next);
#if WITH_KERNEL_VM
/* translate the first buffer */
arch_mmu_query(va, &pa, NULL);
desc->addr = (uint64_t)pa;
/* desc->len is filled in below */
#else
desc->addr = (uint64_t)(uintptr_t)buf;
desc->len = len;
#endif
desc->flags |= write ? 0 : VRING_DESC_F_WRITE; /* mark buffer as write-only if its a block read */
desc->flags |= VRING_DESC_F_NEXT;
#if WITH_KERNEL_VM
/* see if we need to add more descriptors due to scatter gather */
paddr_t next_pa = PAGE_ALIGN(pa + 1);
desc->len = MIN(next_pa - pa, len);
LTRACEF("first descriptor va 0x%lx desc->addr 0x%llx desc->len %u\n", va, desc->addr, desc->len);
len -= desc->len;
while (len > 0) {
/* amount of source buffer handled by this iteration of the loop */
size_t len_tohandle = MIN(len, PAGE_SIZE);
/* translate the next page in the buffer */
va = PAGE_ALIGN(va + 1);
arch_mmu_query(va, &pa, NULL);
LTRACEF("va now 0x%lx, pa 0x%lx, next_pa 0x%lx, remaining len %zu\n", va, pa, next_pa, len);
/* is the new translated physical address contiguous to the last one? */
if (next_pa == pa) {
LTRACEF("extending last one by %zu bytes\n", len_tohandle);
desc->len += len_tohandle;
} else {
uint16_t next_i = virtio_alloc_desc(dev, 0);
struct vring_desc *next_desc = virtio_desc_index_to_desc(dev, 0, next_i);
DEBUG_ASSERT(next_desc);
LTRACEF("doesn't extend, need new desc, allocated desc %i (%p)\n", next_i, next_desc);
/* fill this descriptor in and put it after the last one but before the response descriptor */
next_desc->addr = (uint64_t)pa;
next_desc->len = len_tohandle;
next_desc->flags = write ? 0 : VRING_DESC_F_WRITE; /* mark buffer as write-only if its a block read */
next_desc->flags |= VRING_DESC_F_NEXT;
next_desc->next = desc->next;
desc->next = next_i;
desc = next_desc;
}
len -= len_tohandle;
next_pa += PAGE_SIZE;
}
#endif
/* set up the descriptor pointing to the response */
desc = virtio_desc_index_to_desc(dev, 0, desc->next);
desc->addr = bdev->blk_response_phys;
desc->len = 1;
desc->flags = VRING_DESC_F_WRITE;
/* submit the transfer */
virtio_submit_chain(dev, 0, i);
/* kick it off */
virtio_kick(dev, 0);
/* wait for the transfer to complete */
event_wait(&bdev->io_event);
LTRACEF("status 0x%hhx\n", bdev->blk_response);
mutex_release(&bdev->lock);
return len;
}
status_t virtio_alloc_ring(struct virtio_device *dev, uint index, uint16_t len)
{
LTRACEF("dev %p, index %u, len %u\n", dev, index, len);
DEBUG_ASSERT(dev);
DEBUG_ASSERT(len > 0 && ispow2(len));
DEBUG_ASSERT(index < MAX_VIRTIO_RINGS);
if (len == 0 || !ispow2(len))
return ERR_INVALID_ARGS;
struct vring *ring = &dev->ring[index];
/* allocate a ring */
size_t size = vring_size(len, PAGE_SIZE);
LTRACEF("need %zu bytes\n", size);
#if WITH_KERNEL_VM
void *vptr;
status_t err = vmm_alloc_contiguous(vmm_get_kernel_aspace(), "virtio_ring", size, &vptr, 0, 0, ARCH_MMU_FLAG_UNCACHED_DEVICE);
if (err < 0)
return ERR_NO_MEMORY;
LTRACEF("allocated virtio_ring at va %p\n", vptr);
/* compute the physical address */
paddr_t pa;
err = arch_mmu_query((vaddr_t)vptr, &pa, NULL);
if (err < 0) {
return ERR_NO_MEMORY;
}
LTRACEF("virtio_ring at pa 0x%lx\n", pa);
#else
void *vptr = memalign(PAGE_SIZE, size);
if (!vptr)
return ERR_NO_MEMORY;
LTRACEF("ptr %p\n", vptr);
memset(vptr, 0, size);
/* compute the physical address */
paddr_t pa = (paddr_t)vptr;
#endif
/* initialize the ring */
vring_init(ring, len, vptr, PAGE_SIZE);
dev->ring[index].free_list = 0xffff;
dev->ring[index].free_count = 0;
/* add all the descriptors to the free list */
for (uint i = 0; i < len; i++) {
virtio_free_desc(dev, index, i);
}
/* register the ring with the device */
DEBUG_ASSERT(dev->mmio_config);
dev->mmio_config->guest_page_size = PAGE_SIZE;
dev->mmio_config->queue_sel = index;
dev->mmio_config->queue_num = len;
dev->mmio_config->queue_align = PAGE_SIZE;
dev->mmio_config->queue_pfn = pa / PAGE_SIZE;
/* mark the ring active */
dev->active_rings_bitmap |= (1 << index);
return NO_ERROR;
}
static int do_boot(lkb_t *lkb, size_t len, const char **result)
{
LTRACEF("lkb %p, len %zu, result %p\n", lkb, len, result);
void *buf;
paddr_t buf_phys;
if (vmm_alloc_contiguous(vmm_get_kernel_aspace(), "lkboot_iobuf",
len, &buf, log2_uint(1024*1024), 0, ARCH_MMU_FLAG_UNCACHED) < 0) {
*result = "not enough memory";
return -1;
}
arch_mmu_query((vaddr_t)buf, &buf_phys, NULL);
LTRACEF("iobuffer %p (phys 0x%lx)\n", buf, buf_phys);
if (lkb_read(lkb, buf, len)) {
*result = "io error";
// XXX free buffer here
return -1;
}
/* construct a boot argument list */
const size_t bootargs_size = PAGE_SIZE;
#if 0
void *args = (void *)((uintptr_t)lkb_iobuffer + lkb_iobuffer_size - bootargs_size);
paddr_t args_phys = lkb_iobuffer_phys + lkb_iobuffer_size - bootargs_size;
#elif PLATFORM_ZYNQ
/* grab the top page of sram */
/* XXX do this better */
paddr_t args_phys = SRAM_BASE + SRAM_SIZE - bootargs_size;
void *args = paddr_to_kvaddr(args_phys);
#else
#error need better way
#endif
LTRACEF("boot args %p, phys 0x%lx, len %zu\n", args, args_phys, bootargs_size);
bootargs_start(args, bootargs_size);
bootargs_add_command_line(args, bootargs_size, "what what");
arch_clean_cache_range((vaddr_t)args, bootargs_size);
ulong lk_args[4];
bootargs_generate_lk_arg_values(args_phys, lk_args);
const void *ptr;
/* sniff it to see if it's a bootimage or a raw image */
bootimage_t *bi;
if (bootimage_open(buf, len, &bi) >= 0) {
size_t len;
/* it's a bootimage */
TRACEF("detected bootimage\n");
/* find the lk image */
if (bootimage_get_file_section(bi, TYPE_LK, &ptr, &len) >= 0) {
TRACEF("found lk section at %p\n", ptr);
/* add the boot image to the argument list */
size_t bootimage_size;
bootimage_get_range(bi, NULL, &bootimage_size);
bootargs_add_bootimage_pointer(args, bootargs_size, "pmem", buf_phys, bootimage_size);
}
} else {
/* raw image, just chain load it directly */
TRACEF("raw image, chainloading\n");
ptr = buf;
}
/* start a boot thread to complete the startup */
static struct chainload_args cl_args;
cl_args.func = (void *)ptr;
cl_args.args[0] = lk_args[0];
cl_args.args[1] = lk_args[1];
cl_args.args[2] = lk_args[2];
cl_args.args[3] = lk_args[3];
thread_resume(thread_create("boot", &chainload_thread, &cl_args,
DEFAULT_PRIORITY, DEFAULT_STACK_SIZE));
return 0;
}
static int cmd_display_mem(int argc, const cmd_args *argv)
{
/* save the last address and len so we can continue where we left off */
static unsigned long address;
static size_t len;
if (argc < 3 && len == 0) {
printf("not enough arguments\n");
printf("%s [address] [length]\n", argv[0].str);
return -1;
}
int size;
if (strcmp(argv[0].str, "dw") == 0) {
size = 4;
} else if (strcmp(argv[0].str, "dh") == 0) {
size = 2;
} else {
size = 1;
}
if (argc >= 2) {
address = argv[1].u;
}
if (argc >= 3) {
len = argv[2].u;
}
unsigned long stop = address + len;
int count = 0;
if ((address & (size - 1)) != 0) {
printf("unaligned address, cannot display\n");
return -1;
}
#if WITH_KERNEL_VM
/* preflight the start address to see if it's mapped */
if (arch_mmu_query((vaddr_t)address, NULL, NULL) < 0) {
printf("ERROR: address 0x%lx is unmapped\n", address);
return -1;
}
#endif
for ( ; address < stop; address += size) {
if (count == 0)
printf("0x%08lx: ", address);
switch (size) {
case 4:
printf("%08x ", *(uint32_t *)address);
break;
case 2:
printf("%04hx ", *(uint16_t *)address);
break;
case 1:
printf("%02hhx ", *(uint8_t *)address);
break;
}
count += size;
if (count == 16) {
printf("\n");
count = 0;
}
}
if (count != 0)
printf("\n");
return 0;
}
ssize_t virtio_block_read_write(struct virtio_device *dev, void *buf, off_t offset, size_t len, bool write)
{
struct virtio_block_dev *bdev = (struct virtio_block_dev *)dev->priv;
uint16_t i;
struct vring_desc *desc;
paddr_t pa;
LTRACEF("dev %p, buf %p, offset 0x%llx, len %zu\n", dev, buf, offset, len);
mutex_acquire(&bdev->lock);
/* set up the request */
bdev->blk_req->type = write ? VIRTIO_BLK_T_OUT : VIRTIO_BLK_T_IN;
bdev->blk_req->ioprio = 0;
bdev->blk_req->sector = offset / 512;
LTRACEF("blk_req type %u ioprio %u sector %llu\n",
bdev->blk_req->type, bdev->blk_req->ioprio, bdev->blk_req->sector);
/* put together a transfer */
desc = virtio_alloc_desc_chain(dev, 0, 3, &i);
LTRACEF("after alloc chain desc %p, i %u\n", desc, i);
// XXX not cache safe.
// At the moment only tested on arm qemu, which doesn't emulate cache.
/* set up the descriptor pointing to the head */
desc->addr = bdev->blk_req_phys;
desc->len = sizeof(struct virtio_blk_req);
desc->flags |= VRING_DESC_F_NEXT;
/* set up the descriptor pointing to the buffer */
desc = virtio_desc_index_to_desc(dev, 0, desc->next);
#if WITH_KERNEL_VM
// XXX handle bufs that cross page boundaries
arch_mmu_query((vaddr_t)buf, &pa, NULL);
desc->addr = (uint64_t)pa;
#else
desc->addr = (uint64_t)(uintptr_t)buf;
#endif
desc->len = len;
desc->flags |= write ? 0 : VRING_DESC_F_WRITE; /* mark buffer as write-only if its a block read */
desc->flags |= VRING_DESC_F_NEXT;
/* set up the descriptor pointing to the response */
desc = virtio_desc_index_to_desc(dev, 0, desc->next);
desc->addr = bdev->blk_response_phys;
desc->len = 1;
desc->flags = VRING_DESC_F_WRITE;
/* submit the transfer */
virtio_submit_chain(dev, 0, i);
/* kick it off */
virtio_kick(dev, 0);
/* wait for the transfer to complete */
event_wait(&bdev->io_event);
LTRACEF("status 0x%hhx\n", bdev->blk_response);
mutex_release(&bdev->lock);
return len;
}
