static int blk_dump(struct dumperinfo *di, vm_paddr_t pa, vm_size_t size) { vm_size_t pos, rsz; vm_offset_t va; int c, counter, error, twiddle; printf(" chunk at %#lx: %ld bytes ", (u_long)pa, (long)size); va = 0L; error = counter = twiddle = 0; for (pos = 0; pos < size; pos += MAXDUMPSZ, counter++) { if (counter % 128 == 0) printf("%c\b", "|/-\\"[twiddle++ & 3]); rsz = size - pos; rsz = (rsz > MAXDUMPSZ) ? MAXDUMPSZ : rsz; va = TLB_PHYS_TO_DIRECT(pa + pos); error = dump_write(di, (void *)va, 0, dumplo, rsz); if (error) break; dumplo += rsz; /* Check for user abort. */ c = cncheckc(); if (c == 0x03) return (ECANCELED); if (c != -1) printf("(CTRL-C to abort) "); } printf("... %s\n", (error) ? "fail" : "ok"); return (error); }
void * uma_small_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) { static vm_pindex_t color; vm_paddr_t pa; vm_page_t m; int pflags; void *va; *flags = UMA_SLAB_PRIV; if ((wait & (M_NOWAIT|M_USE_RESERVE)) == M_NOWAIT) pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED; else pflags = VM_ALLOC_SYSTEM | VM_ALLOC_WIRED; if (wait & M_ZERO) pflags |= VM_ALLOC_ZERO; for (;;) { m = vm_page_alloc(NULL, color++, pflags | VM_ALLOC_NOOBJ); if (m == NULL) { if (wait & M_NOWAIT) return (NULL); else VM_WAIT; } else break; } pa = VM_PAGE_TO_PHYS(m); va = (void *)TLB_PHYS_TO_DIRECT(pa); if ((wait & M_ZERO) && ((m->flags & PG_ZERO) == 0)) hwblkclr((void *)TLB_PHYS_TO_DIRECT(pa), PAGE_SIZE); return (va); }
void cpu_thread_alloc(struct thread *td) { struct pcb *pcb; pcb = (struct pcb *)((td->td_kstack + td->td_kstack_pages * PAGE_SIZE - sizeof(struct pcb)) & ~0x3fUL); pcb->pcb_kstack = (uint64_t)(((char *)pcb) - (CCFSZ + SPOFF)); pcb->pcb_nsaved = 0; td->td_frame = (struct trapframe *)pcb - 1; pcb = (struct pcb *)TLB_PHYS_TO_DIRECT(vtophys((vm_offset_t)pcb)); KASSERT(pcb > (struct pcb *)VM_MIN_DIRECT_ADDRESS,("pcb is NULL")); td->td_pcb = pcb; }
void * uma_small_alloc(uma_zone_t zone, vm_size_t bytes, u_int8_t *flags, int wait) { vm_paddr_t pa; vm_page_t m; int pflags; void *va; PMAP_STATS_INC(uma_nsmall_alloc); *flags = UMA_SLAB_PRIV; pflags = malloc2vm_flags(wait) | VM_ALLOC_WIRED; for (;;) { m = vm_page_alloc(NULL, 0, pflags | VM_ALLOC_NOOBJ); if (m == NULL) { if (wait & M_NOWAIT) return (NULL); else VM_WAIT; } else break; } pa = VM_PAGE_TO_PHYS(m); if (dcache_color_ignore == 0 && m->md.color != DCACHE_COLOR(pa)) { KASSERT(m->md.colors[0] == 0 && m->md.colors[1] == 0, ("uma_small_alloc: free page %p still has mappings!", m)); PMAP_STATS_INC(uma_nsmall_alloc_oc); m->md.color = DCACHE_COLOR(pa); dcache_page_inval(pa); } va = (void *)TLB_PHYS_TO_DIRECT(pa); if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0) cpu_block_zero(va, PAGE_SIZE); return (va); }
/* * Implement uiomove(9) from physical memory using a combination * of the direct mapping and sf_bufs to reduce the creation and * destruction of ephemeral mappings. */ int uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio) { struct sf_buf *sf; struct thread *td = curthread; struct iovec *iov; void *cp; vm_offset_t page_offset; vm_paddr_t pa; vm_page_t m; size_t cnt; int error = 0; int save = 0; KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE, ("uiomove_fromphys: mode")); KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == curthread, ("uiomove_fromphys proc")); save = td->td_pflags & TDP_DEADLKTREAT; td->td_pflags |= TDP_DEADLKTREAT; while (n > 0 && uio->uio_resid) { iov = uio->uio_iov; cnt = iov->iov_len; if (cnt == 0) { uio->uio_iov++; uio->uio_iovcnt--; continue; } if (cnt > n) cnt = n; page_offset = offset & PAGE_MASK; cnt = ulmin(cnt, PAGE_SIZE - page_offset); m = ma[offset >> PAGE_SHIFT]; pa = VM_PAGE_TO_PHYS(m); if (m->md.color != DCACHE_COLOR(pa)) { sf = sf_buf_alloc(m, 0); cp = (char *)sf_buf_kva(sf) + page_offset; } else { sf = NULL; cp = (char *)TLB_PHYS_TO_DIRECT(pa) + page_offset; } switch (uio->uio_segflg) { case UIO_USERSPACE: if (ticks - PCPU_GET(switchticks) >= hogticks) uio_yield(); if (uio->uio_rw == UIO_READ) error = copyout(cp, iov->iov_base, cnt); else error = copyin(iov->iov_base, cp, cnt); if (error) { if (sf != NULL) sf_buf_free(sf); goto out; } break; case UIO_SYSSPACE: if (uio->uio_rw == UIO_READ) bcopy(cp, iov->iov_base, cnt); else bcopy(iov->iov_base, cp, cnt); break; case UIO_NOCOPY: break; } if (sf != NULL) sf_buf_free(sf); iov->iov_base = (char *)iov->iov_base + cnt; iov->iov_len -= cnt; uio->uio_resid -= cnt; uio->uio_offset += cnt; offset += cnt; n -= cnt; } out: if (save == 0) td->td_pflags &= ~TDP_DEADLKTREAT; return (error); }
/* * Finish a fork operation, with process p2 nearly set up. * Copy and update the pcb, set up the stack so that the child * ready to run and return to user mode. */ void cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) { struct trapframe *tf; struct frame *fp; struct pcb *pcb1; struct pcb *pcb2, *pcb2orig; vm_offset_t sp; int error; int i; KASSERT(td1 == curthread || td1 == &thread0, ("cpu_fork: p1 not curproc and not proc0")); if ((flags & RFPROC) == 0) return; p2->p_md.md_sigtramp = td1->td_proc->p_md.md_sigtramp; p2->p_md.md_utrap = utrap_hold(td1->td_proc->p_md.md_utrap); /* The pcb must be aligned on a 64-byte boundary. */ pcb1 = td1->td_pcb; pcb2orig = (struct pcb *)((td2->td_kstack + td2->td_kstack_pages * PAGE_SIZE - sizeof(struct pcb)) & ~0x3fUL); pcb2 = (struct pcb *)TLB_PHYS_TO_DIRECT(vtophys((vm_offset_t)pcb2orig)); td2->td_pcb = pcb2; /* * Ensure that p1's pcb is up to date. */ critical_enter(); if ((td1->td_frame->tf_fprs & FPRS_FEF) != 0) savefpctx(pcb1->pcb_ufp); critical_exit(); /* Make sure the copied windows are spilled. */ flushw(); /* Copy the pcb (this will copy the windows saved in the pcb, too). */ bcopy(pcb1, pcb2, sizeof(*pcb1)); /* * If we're creating a new user process and we're sharing the address * space, the parent's top most frame must be saved in the pcb. The * child will pop the frame when it returns to user mode, and may * overwrite it with its own data causing much suffering for the * parent. We check if its already in the pcb, and if not copy it * in. Its unlikely that the copyin will fail, but if so there's not * much we can do. The parent will likely crash soon anyway in that * case. */ if ((flags & RFMEM) != 0 && td1 != &thread0) { sp = td1->td_frame->tf_sp; for (i = 0; i < pcb1->pcb_nsaved; i++) { if (pcb1->pcb_rwsp[i] == sp) break; } if (i == pcb1->pcb_nsaved) { error = copyin((caddr_t)sp + SPOFF, &pcb1->pcb_rw[i], sizeof(struct rwindow)); if (error == 0) { pcb1->pcb_rwsp[i] = sp; pcb1->pcb_nsaved++; } } } /* * Create a new fresh stack for the new process. * Copy the trap frame for the return to user mode as if from a * syscall. This copies most of the user mode register values. */ tf = (struct trapframe *)pcb2orig - 1; bcopy(td1->td_frame, tf, sizeof(*tf)); tf->tf_out[0] = 0; /* Child returns zero */ tf->tf_out[1] = 0; tf->tf_tstate &= ~TSTATE_XCC_C; /* success */ tf->tf_fprs = 0; tf->tf_wstate = WSTATE_U64; td2->td_frame = tf; fp = (struct frame *)tf - 1; fp->fr_local[0] = (u_long)fork_return; fp->fr_local[1] = (u_long)td2; fp->fr_local[2] = (u_long)tf; /* Terminate stack traces at this frame. */ fp->fr_pc = fp->fr_fp = 0; pcb2->pcb_sp = (u_long)fp - SPOFF; pcb2->pcb_pc = (u_long)fork_trampoline - 8; pcb2->pcb_kstack = (uint64_t)(((char *)pcb2orig) - (CCFSZ + SPOFF)); /* Setup to release spin count in fork_exit(). */ td2->td_md.md_spinlock_count = 1; td2->td_md.md_saved_pil = 0; /* * Now, cpu_switch() can schedule the new process. */ }
/* ARGSUSED */ int memrw(struct cdev *dev, struct uio *uio, int flags) { struct iovec *iov; vm_offset_t eva; vm_offset_t off; vm_offset_t ova; vm_offset_t va; vm_prot_t prot; vm_paddr_t pa; vm_size_t cnt; vm_page_t m; int error; int i; uint32_t colors; cnt = 0; colors = 1; error = 0; ova = 0; GIANT_REQUIRED; while (uio->uio_resid > 0 && error == 0) { iov = uio->uio_iov; if (iov->iov_len == 0) { uio->uio_iov++; uio->uio_iovcnt--; if (uio->uio_iovcnt < 0) panic("memrw"); continue; } if (dev2unit(dev) == CDEV_MINOR_MEM) { pa = uio->uio_offset & ~PAGE_MASK; if (!is_physical_memory(pa)) { error = EFAULT; break; } off = uio->uio_offset & PAGE_MASK; cnt = PAGE_SIZE - ((vm_offset_t)iov->iov_base & PAGE_MASK); cnt = ulmin(cnt, PAGE_SIZE - off); cnt = ulmin(cnt, iov->iov_len); m = NULL; for (i = 0; phys_avail[i] != 0; i += 2) { if (pa >= phys_avail[i] && pa < phys_avail[i + 1]) { m = PHYS_TO_VM_PAGE(pa); break; } } if (m != NULL) { if (ova == 0) { if (dcache_color_ignore == 0) colors = DCACHE_COLORS; ova = kmem_alloc_wait(kernel_map, PAGE_SIZE * colors); } if (colors != 1 && m->md.color != -1) va = ova + m->md.color * PAGE_SIZE; else va = ova; pmap_qenter(va, &m, 1); error = uiomove((void *)(va + off), cnt, uio); pmap_qremove(va, 1); } else { va = TLB_PHYS_TO_DIRECT(pa); error = uiomove((void *)(va + off), cnt, uio); } break; } else if (dev2unit(dev) == CDEV_MINOR_KMEM) { va = trunc_page(uio->uio_offset); eva = round_page(uio->uio_offset + iov->iov_len); /* * Make sure that all of the pages are currently * resident so we don't create any zero fill pages. */ for (; va < eva; va += PAGE_SIZE) if (pmap_kextract(va) == 0) return (EFAULT); prot = (uio->uio_rw == UIO_READ) ? VM_PROT_READ : VM_PROT_WRITE; va = uio->uio_offset; if (va < VM_MIN_DIRECT_ADDRESS && kernacc((void *)va, iov->iov_len, prot) == FALSE) return (EFAULT); error = uiomove((void *)va, iov->iov_len, uio); break; } /* else panic! */ } if (ova != 0) kmem_free_wakeup(kernel_map, ova, PAGE_SIZE * colors); return (error); }