int vmfree(void *addr, unsigned long size, int type) { struct filemap *fm = NULL; int pages = PAGES(size); int i, rc; char *vaddr; if (size == 0) return 0; addr = (void *) PAGEADDR(addr); if (!valid_range(addr, size)) return -EINVAL; if (type & (MEM_DECOMMIT | MEM_RELEASE)) { vaddr = (char *) addr; for (i = 0; i < pages; i++) { if (page_directory_mapped(vaddr)) { pte_t flags = get_page_flags(vaddr); unsigned long pfn = BTOP(virt2phys(vaddr)); if (flags & PT_FILE) { handle_t h = (flags & PT_PRESENT) ? pfdb[pfn].owner : pfn; struct filemap *newfm = (struct filemap *) hlookup(h); if (newfm != fm) { if (fm) { if (fm->pages == 0) { rc = free_filemap(fm); } else { rc = unlock_filemap(fm); } if (rc < 0) return rc; } fm = newfm; rc = wait_for_object(fm, INFINITE); if (rc < 0) return rc; } fm->pages--; unmap_page(vaddr); if (flags & PT_PRESENT) free_pageframe(pfn); } else if (flags & PT_PRESENT) { unmap_page(vaddr); free_pageframe(pfn); } } vaddr += PAGESIZE; } } if (fm) { if (fm->pages == 0) { rc = free_filemap(fm); } else { rc = unlock_filemap(fm); } if (rc < 0) return rc; } else if (type & MEM_RELEASE) { rmap_free(vmap, BTOP(addr), pages); } return 0; }
void *vmmap(void *addr, unsigned long size, int protect, struct file *filp, off64_t offset, int *rc) { int pages = PAGES(size); unsigned long flags = pte_flags_from_protect(protect); struct filemap *fm; int i; char *vaddr; if (rc) *rc = 0; if (size == 0 || flags == 0xFFFFFFFF) { if (rc) *rc = -EINVAL; return NULL; } addr = (void *) PAGEADDR(addr); if (addr == NULL) { addr = (void *) PTOB(rmap_alloc(vmap, pages)); if (addr == NULL) { if (rc) *rc = -ENOMEM; return NULL; } } else { if (rmap_reserve(vmap, BTOP(addr), pages)) { if (rc) *rc = -ENOMEM; return NULL; } } fm = (struct filemap *) kmalloc(sizeof(struct filemap)); if (!fm) { rmap_free(vmap, BTOP(addr), pages); if (rc) *rc = -ENOMEM; return NULL; } init_object(&fm->object, OBJECT_FILEMAP); fm->self = halloc(&fm->object); fm->file = halloc(&filp->iob.object); if (fm->self < 0 || fm->file < 0) { if (rc) *rc = -ENFILE; return NULL; } hprotect(fm->self); hprotect(fm->file); fm->offset = offset; fm->pages = pages; fm->object.signaled = 1; fm->addr = addr; fm->size = size; fm->protect = flags | PT_FILE; vaddr = (char *) addr; flags = (flags & ~PT_USER) | PT_FILE; for (i = 0; i < pages; i++) { map_page(vaddr, fm->self, flags); vaddr += PAGESIZE; } return addr; }
void *kmalloc_tag(int size, unsigned long tag) { struct bucket *b; int bucket; void *addr; // Handle large allocation by allocating pages if (size > PAGESIZE / 2) { // Allocate pages addr = alloc_pages(PAGES(size), tag ? tag : 'ALOC'); // Set size in pfn entry pfdb[BTOP(virt2phys(addr))].size = PAGES(size) + PAGESHIFT; return addr; } // Otherwise allocate from one of the buckets bucket = BUCKET(size); b = &buckets[bucket]; // If bucket is empty the allocate one more page for the bucket if (b->mem == 0) { char *p; int i; // Allocate new page addr = alloc_pages(1, 'HEAP'); // Set bucket number in pfn entry pfdb[BTOP(virt2phys(addr))].size = bucket; // Split page into chunks p = (char *) addr; for (i = 0; i < PAGESIZE; i += b->size) { *(void **)(p + i) = b->mem; b->mem = p + i; } // Update count of pages used for this bucket b->pages++; } // Allocate chunk from bucket addr = b->mem; b->mem = *(void **) addr; // Return allocated chunk return addr; }
void miounmap(void *addr, int size) { int i; int pages = PAGES(size); for (i = 0; i < pages; i++) unmap_page((char *) addr + PTOB(i)); rmap_free(vmap, BTOP(addr), pages); }
static int valid_range(void *addr, int size) { int pages = PAGES(size); if ((unsigned long) addr < VMEM_START) return 0; if (KERNELSPACE((unsigned long) addr + pages * PAGESIZE)) return 0; if (rmap_status(vmap, BTOP(addr), pages) != 1) return 0; return 1; }
/*ARGSUSED2*/ static int mmmmap(dev_t dev, off_t off, int prot) { pfn_t pf; struct memlist *pmem; minor_t minor = getminor(dev); switch (minor) { case M_MEM: pf = btop(off); memlist_read_lock(); for (pmem = phys_install; pmem != NULL; pmem = pmem->next) { if (pf >= BTOP(pmem->address) && pf < BTOP(pmem->address + pmem->size)) { memlist_read_unlock(); return (impl_obmem_pfnum(pf)); } } memlist_read_unlock(); break; case M_KMEM: case M_ALLKMEM: /* no longer supported with KPR */ return (-1); case M_ZERO: /* * We shouldn't be mmap'ing to /dev/zero here as * mmsegmap() should have already converted * a mapping request for this device to a mapping * using seg_vn for anonymous memory. */ break; } return (-1); }
static int free_filemap(struct filemap *fm) { int rc; hunprotect(fm->file); rc = hfree(fm->file); if (rc < 0) return rc; rmap_free(vmap, BTOP(fm->addr), PAGES(fm->size)); hunprotect(fm->self); rc = hfree(fm->self); if (rc < 0) return rc; return 0; }
void *miomap(unsigned long addr, int size, int protect) { char *vaddr; int i; unsigned long flags = pte_flags_from_protect(protect); int pages = PAGES(size); vaddr = (char *) PTOB(rmap_alloc(vmap, pages)); if (vaddr == NULL) return NULL; for (i = 0; i < pages; i++) { map_page(vaddr + PTOB(i), BTOP(addr) + i, flags | PT_PRESENT); } return vaddr; }
int vmsync(void *addr, unsigned long size) { struct filemap *fm = NULL; int pages = PAGES(size); int i, rc; char *vaddr; if (size == 0) return 0; addr = (void *) PAGEADDR(addr); if (!valid_range(addr, size)) return -EINVAL; vaddr = (char *) addr; for (i = 0; i < pages; i++) { if (page_directory_mapped(vaddr)) { pte_t flags = get_page_flags(vaddr); if ((flags & (PT_FILE | PT_PRESENT | PT_DIRTY)) == (PT_FILE | PT_PRESENT | PT_DIRTY)) { unsigned long pfn = BTOP(virt2phys(vaddr)); struct filemap *newfm = (struct filemap *) hlookup(pfdb[pfn].owner); if (newfm != fm) { if (fm) { rc = unlock_filemap(fm); if (rc < 0) return rc; } fm = newfm; rc = wait_for_object(fm, INFINITE); if (rc < 0) return rc; } rc = save_file_page(fm, vaddr); if (rc < 0) return rc; } } vaddr += PAGESIZE; } if (fm) { rc = unlock_filemap(fm); if (rc < 0) return rc; } return 0; }
void kfree(void *addr) { unsigned long bucket; struct bucket *b; // Check for NULL if (!addr) return; // Get page information bucket = pfdb[BTOP(virt2phys(addr))].size; // If a whole page or more, free directly if (bucket >= PAGESHIFT) { free_pages(addr, bucket - PAGESHIFT); return; } // Get bucket b = &buckets[bucket]; // Free chunk to bucket *(void **) addr = b->mem; b->mem = addr; }
unsigned long virt2pfn(void *vaddr) { return BTOP(GET_PTE(vaddr) & PT_PFNMASK); }
/*ARGSUSED3*/ static int mmrw(dev_t dev, struct uio *uio, enum uio_rw rw, cred_t *cred) { pfn_t v; struct iovec *iov; int error = 0; size_t c; ssize_t oresid = uio->uio_resid; minor_t minor = getminor(dev); while (uio->uio_resid > 0 && error == 0) { iov = uio->uio_iov; if (iov->iov_len == 0) { uio->uio_iov++; uio->uio_iovcnt--; if (uio->uio_iovcnt < 0) panic("mmrw"); continue; } switch (minor) { case M_MEM: memlist_read_lock(); if (!address_in_memlist(phys_install, (uint64_t)uio->uio_loffset, 1)) { memlist_read_unlock(); error = EFAULT; break; } memlist_read_unlock(); v = BTOP((u_offset_t)uio->uio_loffset); error = mmio(uio, rw, v, uio->uio_loffset & PAGEOFFSET, 0, NULL); break; case M_KMEM: case M_ALLKMEM: { page_t **ppp = NULL; caddr_t vaddr = (caddr_t)uio->uio_offset; int try_lock = NEED_LOCK_KVADDR(vaddr); int locked = 0; if ((error = plat_mem_do_mmio(uio, rw)) != ENOTSUP) break; /* * If vaddr does not map a valid page, as_pagelock() * will return failure. Hence we can't check the * return value and return EFAULT here as we'd like. * seg_kp and seg_kpm do not properly support * as_pagelock() for this context so we avoid it * using the try_lock set check above. Some day when * the kernel page locking gets redesigned all this * muck can be cleaned up. */ if (try_lock) locked = (as_pagelock(&kas, &ppp, vaddr, PAGESIZE, S_WRITE) == 0); v = hat_getpfnum(kas.a_hat, (caddr_t)(uintptr_t)uio->uio_loffset); if (v == PFN_INVALID) { if (locked) as_pageunlock(&kas, ppp, vaddr, PAGESIZE, S_WRITE); error = EFAULT; break; } error = mmio(uio, rw, v, uio->uio_loffset & PAGEOFFSET, minor == M_ALLKMEM || mm_kmem_io_access, (locked && ppp) ? *ppp : NULL); if (locked) as_pageunlock(&kas, ppp, vaddr, PAGESIZE, S_WRITE); } break; case M_ZERO: if (rw == UIO_READ) { label_t ljb; if (on_fault(&ljb)) { no_fault(); error = EFAULT; break; } uzero(iov->iov_base, iov->iov_len); no_fault(); uio->uio_resid -= iov->iov_len; uio->uio_loffset += iov->iov_len; break; } /* else it's a write, fall through to NULL case */ /*FALLTHROUGH*/ case M_NULL: if (rw == UIO_READ) return (0); c = iov->iov_len; iov->iov_base += c; iov->iov_len -= c; uio->uio_loffset += c; uio->uio_resid -= c; break; } } return (uio->uio_resid == oresid ? error : 0); }
int vmem_proc(struct proc_file *pf, void *arg) { return list_memmap(pf, vmap, BTOP(VMEM_START)); }
void *vmalloc(void *addr, unsigned long size, int type, int protect, unsigned long tag, int *rc) { int pages = PAGES(size); unsigned long flags = pte_flags_from_protect(protect); int i; if (rc) *rc = 0; if (size == 0) { if (rc) *rc = -EINVAL; return NULL; } if ((type & MEM_COMMIT) != 0 && flags == 0xFFFFFFFF) { if (rc) *rc = -EINVAL; return NULL; } addr = (void *) PAGEADDR(addr); if (!addr && (type & MEM_COMMIT) != 0) type |= MEM_RESERVE; if (!tag) tag = 'VM'; if (type & MEM_RESERVE) { if (addr == NULL) { if (type & MEM_ALIGN64K) { addr = (void *) PTOB(rmap_alloc_align(vmap, pages, 64 * 1024 / PAGESIZE)); } else { addr = (void *) PTOB(rmap_alloc(vmap, pages)); } if (addr == NULL) { if (rc) *rc = -ENOMEM; return NULL; } } else { if (rmap_reserve(vmap, BTOP(addr), pages)) { if (rc) *rc = -ENOMEM; return NULL; } } } else { if (!valid_range(addr, size)) { if (rc) *rc = -EFAULT; return NULL; } } if (type & MEM_COMMIT) { char *vaddr; unsigned long pfn; vaddr = (char *) addr; for (i = 0; i < pages; i++) { if (page_mapped(vaddr)) { set_page_flags(vaddr, flags | PT_PRESENT); } else { pfn = alloc_pageframe(tag); if (pfn == 0xFFFFFFFF) { if (rc) *rc = -ENOMEM; return NULL; } map_page(vaddr, pfn, flags | PT_PRESENT); memset(vaddr, 0, PAGESIZE); } vaddr += PAGESIZE; } } return addr; }
void init_vmm() { vmap = (struct rmap *) kmalloc(VMAP_ENTRIES * sizeof(struct rmap)); rmap_init(vmap, VMAP_ENTRIES); rmap_free(vmap, BTOP(VMEM_START), BTOP(OSBASE - VMEM_START)); }
/* * Just pass in an unused filename. */ void cmd_snap(void) { int c, fd, n; physaddr_t paddr; size_t offset; char *buf; char *filename; struct node_table *nt; int type; char *elf_header; Elf64_Phdr *load; int load_index; if (!supported) error(FATAL, "command not supported on the %s architecture\n", pc->machine_type); filename = NULL; buf = GETBUF(PAGESIZE()); type = KDUMP_ELF64; while ((c = getopt(argcnt, args, "n")) != EOF) { switch(c) { case 'n': if (machine_type("X86_64")) option_not_supported('n'); else type = NETDUMP_ELF64; break; default: argerrs++; break; } } if (argerrs || !args[optind]) cmd_usage(pc->curcmd, SYNOPSIS); while (args[optind]) { if (filename) cmd_usage(pc->curcmd, SYNOPSIS); if (file_exists(args[optind], NULL)) error(FATAL, "%s: file already exists\n", args[optind]); else if ((fd = open(args[optind], O_RDWR|O_CREAT, 0644)) < 0) error(FATAL, args[optind]); filename = args[optind]; optind++; } if (!filename) cmd_usage(pc->curcmd, SYNOPSIS); init_ram_segments(); if (!(elf_header = generate_elf_header(type, fd, filename))) error(FATAL, "cannot generate ELF header\n"); load = (Elf64_Phdr *)(elf_header + sizeof(Elf64_Ehdr) + sizeof(Elf64_Phdr)); load_index = machine_type("X86_64") || machine_type("IA64") ? 1 : 0; for (n = 0; n < vt->numnodes; n++) { nt = &vt->node_table[n]; paddr = nt->start_paddr; offset = load[load_index + n].p_offset; for (c = 0; c < nt->size; c++, paddr += PAGESIZE()) { if (!verify_paddr(paddr)) continue; if (!readmem(paddr, PHYSADDR, &buf[0], PAGESIZE(), "memory page", QUIET|RETURN_ON_ERROR)) continue; lseek(fd, (off_t)(paddr + offset - nt->start_paddr), SEEK_SET); if (write(fd, &buf[0], PAGESIZE()) != PAGESIZE()) error(FATAL, "write to dumpfile failed\n"); if (!print_progress(filename, BTOP(paddr))) return; } } fprintf(stderr, "\r%s: [100%%] ", filename); fprintf(fp, "\n"); sprintf(buf, "/bin/ls -l %s\n", filename); system(buf); FREEBUF(elf_header); FREEBUF(buf); }