/*===========================================================================* * do_mmap * *===========================================================================*/ PUBLIC int do_mmap(message *m) { int r, n; struct vmproc *vmp; int mfflags = 0; struct vir_region *vr = NULL; if((r=vm_isokendpt(m->m_source, &n)) != OK) { panic("do_mmap: message from strange source: %d", m->m_source); } vmp = &vmproc[n]; if(!(vmp->vm_flags & VMF_HASPT)) return ENXIO; if(m->VMM_FD == -1 || (m->VMM_FLAGS & MAP_ANON)) { int s; vir_bytes v; u32_t vrflags = VR_ANON | VR_WRITABLE; size_t len = (vir_bytes) m->VMM_LEN; if(m->VMM_FD != -1) { return EINVAL; } /* Contiguous phys memory has to be preallocated. */ if((m->VMM_FLAGS & (MAP_CONTIG|MAP_PREALLOC)) == MAP_CONTIG) { return EINVAL; } if(m->VMM_FLAGS & MAP_PREALLOC) mfflags |= MF_PREALLOC; if(m->VMM_FLAGS & MAP_LOWER16M) vrflags |= VR_LOWER16MB; if(m->VMM_FLAGS & MAP_LOWER1M) vrflags |= VR_LOWER1MB; if(m->VMM_FLAGS & MAP_ALIGN64K) vrflags |= VR_PHYS64K; if(m->VMM_FLAGS & MAP_SHARED) vrflags |= VR_SHARED; if(m->VMM_FLAGS & MAP_CONTIG) vrflags |= VR_CONTIG; if(len % VM_PAGE_SIZE) len += VM_PAGE_SIZE - (len % VM_PAGE_SIZE); if(!(vr = map_page_region(vmp, arch_vir2map(vmp, m->VMM_ADDR ? m->VMM_ADDR : vmp->vm_stacktop), VM_DATATOP, len, MAP_NONE, vrflags, mfflags))) { return ENOMEM; } } else { return ENOSYS; } /* Return mapping, as seen from process. */ assert(vr); m->VMM_RETADDR = arch_map2vir(vmp, vr->vaddr); return OK; }
/*===========================================================================* * do_remap * *===========================================================================*/ PUBLIC int do_remap(message *m) { int dn, sn; vir_bytes da, sa, startv; size_t size; struct vir_region *vr, *region; struct vmproc *dvmp, *svmp; int r; da = (vir_bytes) m->VMRE_DA; sa = (vir_bytes) m->VMRE_SA; size = m->VMRE_SIZE; if ((r = vm_isokendpt((endpoint_t) m->VMRE_D, &dn)) != OK) return EINVAL; if ((r = vm_isokendpt((endpoint_t) m->VMRE_S, &sn)) != OK) return EINVAL; dvmp = &vmproc[dn]; svmp = &vmproc[sn]; /* da is not translated by arch_vir2map(), * it's handled a little differently, * since in map_remap(), we have to know * about whether the user needs to bind to * THAT address or be chosen by the system. */ sa = arch_vir2map(svmp, sa); if (!(region = map_lookup(svmp, sa))) return EINVAL; if(region->vaddr != sa) { printf("VM: do_remap: not start of region.\n"); return EFAULT; } if(!(region->flags & VR_SHARED)) { printf("VM: do_remap: not shared.\n"); return EFAULT; } if (size % VM_PAGE_SIZE) size += VM_PAGE_SIZE - size % VM_PAGE_SIZE; if(size != region->length) { printf("VM: do_remap: not size of region.\n"); return EFAULT; } if ((r = map_remap(dvmp, da, size, region, &startv)) != OK) return r; m->VMRE_RETA = (char *) arch_map2vir(dvmp, startv); return OK; }
/*===========================================================================* * do_map_phys * *===========================================================================*/ PUBLIC int do_map_phys(message *m) { int r, n; struct vmproc *vmp; endpoint_t target; struct vir_region *vr; vir_bytes len; phys_bytes startaddr; size_t offset; target = m->VMMP_EP; len = m->VMMP_LEN; if (len <= 0) return EINVAL; if(target == SELF) target = m->m_source; if((r=vm_isokendpt(target, &n)) != OK) return EINVAL; startaddr = (vir_bytes)m->VMMP_PHADDR; /* First check permission, then round range down/up. Caller can't * help it if we can't map in lower than page granularity. */ if(map_perm_check(m->m_source, target, startaddr, len) != OK) { printf("VM: unauthorized mapping of 0x%lx by %d\n", startaddr, m->m_source); return EPERM; } vmp = &vmproc[n]; if(!(vmp->vm_flags & VMF_HASPT)) return ENXIO; offset = startaddr % VM_PAGE_SIZE; len += offset; startaddr -= offset; if(len % VM_PAGE_SIZE) len += VM_PAGE_SIZE - (len % VM_PAGE_SIZE); if(!(vr = map_page_region(vmp, arch_vir2map(vmp, vmp->vm_stacktop), VM_DATATOP, len, startaddr, VR_DIRECT | VR_NOPF | VR_WRITABLE, 0))) { return ENOMEM; } m->VMMP_VADDR_REPLY = (void *) (arch_map2vir(vmp, vr->vaddr) + offset); return OK; }
int scall_mmap(kipc_msg_t *m) { int err, n; struct vmproc *vmp; int mfflags = 0; struct vir_region *vr = NULL; if((err = vm_isokendpt(m->m_source, &n)) != 0) { vm_panic("do_mmap: message from strange source", m->m_source); } vmp = &vmproc[n]; if(!(vmp->vm_flags & VMF_HASPT)) return -ENXIO; if(m->VMM_FD == -1 || (m->VMM_FLAGS & MAP_ANONYMOUS)) { int s; vir_bytes v; u32_t vrflags = VR_ANON | VR_WRITABLE; size_t len = (vir_bytes) m->VMM_LEN; if(m->VMM_FD != -1) { return -EINVAL; } if(m->VMM_FLAGS & MAP_CONTIG) mfflags |= MF_CONTIG; if(m->VMM_FLAGS & MAP_PREALLOC) mfflags |= MF_PREALLOC; if(m->VMM_FLAGS & MAP_LOWER16M) vrflags |= VR_LOWER16MB; if(m->VMM_FLAGS & MAP_LOWER1M) vrflags |= VR_LOWER1MB; if(m->VMM_FLAGS & MAP_ALIGN64K) vrflags |= VR_PHYS64K; if(m->VMM_FLAGS & MAP_SHARED) vrflags |= VR_SHARED; if(len % VM_PAGE_SIZE) len += VM_PAGE_SIZE - (len % VM_PAGE_SIZE); if(!(vr = map_page_region(vmp, arch_vir2map(vmp, m->VMM_ADDR ? m->VMM_ADDR : vmp->vm_stacktop), VM_DATATOP, len, MAP_NONE, vrflags, mfflags))) { return -ENOMEM; } } else { return -ENOSYS; } /* Return mapping, as seen from process. */ vm_assert(vr); m->VMM_RETADDR = arch_map2vir(vmp, vr->vaddr); return m->VMM_RETADDR; }
/*===========================================================================* * pt_bind * *===========================================================================*/ PUBLIC int pt_bind(pt_t *pt, struct vmproc *who) { int slot, ispt; u32_t phys; void *pdes; /* Basic sanity checks. */ assert(who); assert(who->vm_flags & VMF_INUSE); assert(pt); assert(pagedir_pde >= 0); slot = who->vm_slot; assert(slot >= 0); assert(slot < ELEMENTS(vmproc)); assert(slot < I386_VM_PT_ENTRIES); phys = pt->pt_dir_phys & I386_VM_ADDR_MASK; assert(pt->pt_dir_phys == phys); /* Update "page directory pagetable." */ page_directories[slot] = phys | I386_VM_PRESENT|I386_VM_WRITE; /* This is where the PDE's will be visible to the kernel * in its address space. */ pdes = (void *) arch_map2vir(&vmproc[VMP_SYSTEM], pagedir_pde*I386_BIG_PAGE_SIZE + slot * I386_PAGE_SIZE); #if 0 printf("VM: slot %d endpoint %d has pde val 0x%lx at kernel address 0x%lx\n", slot, who->vm_endpoint, page_directories[slot], pdes); #endif /* Tell kernel about new page table root. */ return sys_vmctl_set_addrspace(who->vm_endpoint, pt ? pt->pt_dir_phys : 0, pt ? pdes : 0); }
/*===========================================================================* * do_remap * *===========================================================================*/ int do_remap(kipc_msg_t *m) { int d, dn, s, sn; vir_bytes da, sa, startv; size_t size; struct vir_region *vr, *region; struct vmproc *dvmp, *svmp; int r; d = m->VMRE_D; s = m->VMRE_S; da = (vir_bytes) m->VMRE_DA; sa = (vir_bytes) m->VMRE_SA; size = m->VMRE_SIZE; if ((r = vm_isokendpt(d, &dn)) != 0) return -EINVAL; if ((r = vm_isokendpt(s, &sn)) != 0) return -EINVAL; dvmp = &vmproc[dn]; svmp = &vmproc[sn]; /* da is not translated by arch_vir2map(), * it's handled a little differently, * since in map_remap(), we have to know * about whether the user needs to bind to * THAT address or be chosen by the system. */ sa = arch_vir2map(svmp, sa); if (!(region = map_lookup(svmp, sa))) return -EINVAL; if ((r = map_remap(dvmp, da, size, region, &startv)) != 0) return r; m->VMRE_RETA = (char *) arch_map2vir(dvmp, startv); return 0; }
/*===========================================================================* * pt_init * *===========================================================================*/ PUBLIC void pt_init(phys_bytes usedlimit) { /* By default, the kernel gives us a data segment with pre-allocated * memory that then can't grow. We want to be able to allocate memory * dynamically, however. So here we copy the part of the page table * that's ours, so we get a private page table. Then we increase the * hardware segment size so we can allocate memory above our stack. */ pt_t *newpt; int s, r; vir_bytes v; phys_bytes lo, hi; vir_bytes extra_clicks; u32_t moveup = 0; int global_bit_ok = 0; int free_pde; int p; struct vm_ep_data ep_data; vir_bytes sparepages_mem; phys_bytes sparepages_ph; vir_bytes ptr; /* Shorthand. */ newpt = &vmprocess->vm_pt; /* Get ourselves spare pages. */ ptr = (vir_bytes) static_sparepages; ptr += I386_PAGE_SIZE - (ptr % I386_PAGE_SIZE); if(!(sparepages_mem = ptr)) panic("pt_init: aalloc for spare failed"); if((r=sys_umap(SELF, VM_D, (vir_bytes) sparepages_mem, I386_PAGE_SIZE*SPAREPAGES, &sparepages_ph)) != OK) panic("pt_init: sys_umap failed: %d", r); missing_spares = 0; assert(STATIC_SPAREPAGES < SPAREPAGES); for(s = 0; s < SPAREPAGES; s++) { if(s >= STATIC_SPAREPAGES) { sparepages[s].page = NULL; missing_spares++; continue; } sparepages[s].page = (void *) (sparepages_mem + s*I386_PAGE_SIZE); sparepages[s].phys = sparepages_ph + s*I386_PAGE_SIZE; } /* global bit and 4MB pages available? */ global_bit_ok = _cpufeature(_CPUF_I386_PGE); bigpage_ok = _cpufeature(_CPUF_I386_PSE); /* Set bit for PTE's and PDE's if available. */ if(global_bit_ok) global_bit = I386_VM_GLOBAL; /* The kernel and boot time processes need an identity mapping. * We use full PDE's for this without separate page tables. * Figure out which pde we can start using for other purposes. */ id_map_high_pde = usedlimit / I386_BIG_PAGE_SIZE; /* We have to make mappings up till here. */ free_pde = id_map_high_pde+1; /* Initial (current) range of our virtual address space. */ lo = CLICK2ABS(vmprocess->vm_arch.vm_seg[T].mem_phys); hi = CLICK2ABS(vmprocess->vm_arch.vm_seg[S].mem_phys + vmprocess->vm_arch.vm_seg[S].mem_len); assert(!(lo % I386_PAGE_SIZE)); assert(!(hi % I386_PAGE_SIZE)); if(lo < VM_PROCSTART) { moveup = VM_PROCSTART - lo; assert(!(VM_PROCSTART % I386_PAGE_SIZE)); assert(!(lo % I386_PAGE_SIZE)); assert(!(moveup % I386_PAGE_SIZE)); } /* Make new page table for ourselves, partly copied * from the current one. */ if(pt_new(newpt) != OK) panic("pt_init: pt_new failed"); /* Set up mappings for VM process. */ for(v = lo; v < hi; v += I386_PAGE_SIZE) { phys_bytes addr; u32_t flags; /* We have to write the new position in the PT, * so we can move our segments. */ if(pt_writemap(vmprocess, newpt, v+moveup, v, I386_PAGE_SIZE, I386_VM_PRESENT|I386_VM_WRITE|I386_VM_USER, 0) != OK) panic("pt_init: pt_writemap failed"); } /* Move segments up too. */ vmprocess->vm_arch.vm_seg[T].mem_phys += ABS2CLICK(moveup); vmprocess->vm_arch.vm_seg[D].mem_phys += ABS2CLICK(moveup); vmprocess->vm_arch.vm_seg[S].mem_phys += ABS2CLICK(moveup); /* Allocate us a page table in which to remember page directory * pointers. */ if(!(page_directories = vm_allocpage(&page_directories_phys, VMP_PAGETABLE))) panic("no virt addr for vm mappings"); memset(page_directories, 0, I386_PAGE_SIZE); /* Increase our hardware data segment to create virtual address * space above our stack. We want to increase it to VM_DATATOP, * like regular processes have. */ extra_clicks = ABS2CLICK(VM_DATATOP - hi); vmprocess->vm_arch.vm_seg[S].mem_len += extra_clicks; /* We pretend to the kernel we have a huge stack segment to * increase our data segment. */ vmprocess->vm_arch.vm_data_top = (vmprocess->vm_arch.vm_seg[S].mem_vir + vmprocess->vm_arch.vm_seg[S].mem_len) << CLICK_SHIFT; /* Where our free virtual address space starts. * This is only a hint to the VM system. */ newpt->pt_virtop = 0; /* Let other functions know VM now has a private page table. */ vmprocess->vm_flags |= VMF_HASPT; /* Now reserve another pde for kernel's own mappings. */ { int kernmap_pde; phys_bytes addr, len; int flags, index = 0; u32_t offset = 0; kernmap_pde = free_pde++; offset = kernmap_pde * I386_BIG_PAGE_SIZE; while(sys_vmctl_get_mapping(index, &addr, &len, &flags) == OK) { vir_bytes vir; if(index >= MAX_KERNMAPPINGS) panic("VM: too many kernel mappings: %d", index); kern_mappings[index].phys_addr = addr; kern_mappings[index].len = len; kern_mappings[index].flags = flags; kern_mappings[index].lin_addr = offset; kern_mappings[index].flags = I386_VM_PRESENT | I386_VM_USER | I386_VM_WRITE | global_bit; if(flags & VMMF_UNCACHED) kern_mappings[index].flags |= PTF_NOCACHE; if(addr % I386_PAGE_SIZE) panic("VM: addr unaligned: %d", addr); if(len % I386_PAGE_SIZE) panic("VM: len unaligned: %d", len); vir = arch_map2vir(&vmproc[VMP_SYSTEM], offset); if(sys_vmctl_reply_mapping(index, vir) != OK) panic("VM: reply failed"); offset += len; index++; kernmappings++; } } /* Find a PDE below processes available for mapping in the * page directories (readonly). */ pagedir_pde = free_pde++; pagedir_pde_val = (page_directories_phys & I386_VM_ADDR_MASK) | I386_VM_PRESENT | I386_VM_USER | I386_VM_WRITE; /* Tell kernel about free pde's. */ while(free_pde*I386_BIG_PAGE_SIZE < VM_PROCSTART) { if((r=sys_vmctl(SELF, VMCTL_I386_FREEPDE, free_pde++)) != OK) { panic("VMCTL_I386_FREEPDE failed: %d", r); } } /* first pde in use by process. */ proc_pde = free_pde; /* Give our process the new, copied, private page table. */ pt_mapkernel(newpt); /* didn't know about vm_dir pages earlier */ pt_bind(newpt, vmprocess); /* new segment limit for the kernel after paging is enabled */ ep_data.data_seg_limit = free_pde*I386_BIG_PAGE_SIZE; /* the memory map which must be installed after paging is enabled */ ep_data.mem_map = vmprocess->vm_arch.vm_seg; /* Now actually enable paging. */ if(sys_vmctl_enable_paging(&ep_data) != OK) panic("pt_init: enable paging failed"); /* Back to reality - this is where the stack actually is. */ vmprocess->vm_arch.vm_seg[S].mem_len -= extra_clicks; /* Pretend VM stack top is the same as any regular process, not to * have discrepancies with new VM instances later on. */ vmprocess->vm_stacktop = VM_STACKTOP; /* All OK. */ return; }
/*===========================================================================* * vm_allocpage * *===========================================================================*/ PUBLIC void *vm_allocpage(phys_bytes *phys, int reason) { /* Allocate a page for use by VM itself. */ phys_bytes newpage; vir_bytes loc; pt_t *pt; int r; static int level = 0; void *ret; pt = &vmprocess->vm_pt; assert(reason >= 0 && reason < VMP_CATEGORIES); level++; assert(level >= 1); assert(level <= 2); if(level > 1 || !(vmprocess->vm_flags & VMF_HASPT) || !meminit_done) { int r; void *s; s=vm_getsparepage(phys); level--; if(!s) { util_stacktrace(); printf("VM: warning: out of spare pages\n"); } return s; } /* VM does have a pagetable, so get a page and map it in there. * Where in our virtual address space can we put it? */ loc = findhole(pt, arch_vir2map(vmprocess, vmprocess->vm_stacktop), vmprocess->vm_arch.vm_data_top); if(loc == NO_MEM) { level--; printf("VM: vm_allocpage: findhole failed\n"); return NULL; } /* Allocate page of memory for use by VM. As VM * is trusted, we don't have to pre-clear it. */ if((newpage = alloc_mem(CLICKSPERPAGE, 0)) == NO_MEM) { level--; printf("VM: vm_allocpage: alloc_mem failed\n"); return NULL; } *phys = CLICK2ABS(newpage); /* Map this page into our address space. */ if((r=pt_writemap(vmprocess, pt, loc, *phys, I386_PAGE_SIZE, I386_VM_PRESENT | I386_VM_USER | I386_VM_WRITE, 0)) != OK) { free_mem(newpage, CLICKSPERPAGE); printf("vm_allocpage writemap failed\n"); level--; return NULL; } if((r=sys_vmctl(SELF, VMCTL_FLUSHTLB, 0)) != OK) { panic("VMCTL_FLUSHTLB failed: %d", r); } level--; /* Return user-space-ready pointer to it. */ ret = (void *) arch_map2vir(vmprocess, loc); return ret; }