static int
blk_dump(struct dumperinfo *di, vm_paddr_t pa, vm_size_t size)
{
vm_size_t pos, rsz;
vm_offset_t va;
int c, counter, error, twiddle;
printf(" chunk at %#lx: %ld bytes ", (u_long)pa, (long)size);
va = 0L;
error = counter = twiddle = 0;
for (pos = 0; pos < size; pos += MAXDUMPSZ, counter++) {
if (counter % 128 == 0)
printf("%c\b", "|/-\\"[twiddle++ & 3]);
rsz = size - pos;
rsz = (rsz > MAXDUMPSZ) ? MAXDUMPSZ : rsz;
va = TLB_PHYS_TO_DIRECT(pa + pos);
error = dump_write(di, (void *)va, 0, dumplo, rsz);
if (error)
break;
dumplo += rsz;
/* Check for user abort. */
c = cncheckc();
if (c == 0x03)
return (ECANCELED);
if (c != -1)
printf("(CTRL-C to abort) ");
}
printf("... %s\n", (error) ? "fail" : "ok");
return (error);
}
void *
uma_small_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
{
static vm_pindex_t color;
vm_paddr_t pa;
vm_page_t m;
int pflags;
void *va;
*flags = UMA_SLAB_PRIV;
if ((wait & (M_NOWAIT|M_USE_RESERVE)) == M_NOWAIT)
pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED;
else
pflags = VM_ALLOC_SYSTEM | VM_ALLOC_WIRED;
if (wait & M_ZERO)
pflags |= VM_ALLOC_ZERO;
for (;;) {
m = vm_page_alloc(NULL, color++, pflags | VM_ALLOC_NOOBJ);
if (m == NULL) {
if (wait & M_NOWAIT)
return (NULL);
else
VM_WAIT;
} else
break;
}
pa = VM_PAGE_TO_PHYS(m);
va = (void *)TLB_PHYS_TO_DIRECT(pa);
if ((wait & M_ZERO) && ((m->flags & PG_ZERO) == 0))
hwblkclr((void *)TLB_PHYS_TO_DIRECT(pa), PAGE_SIZE);
return (va);
}
void
cpu_thread_alloc(struct thread *td)
{
struct pcb *pcb;
pcb = (struct pcb *)((td->td_kstack + td->td_kstack_pages * PAGE_SIZE -
sizeof(struct pcb)) & ~0x3fUL);
pcb->pcb_kstack = (uint64_t)(((char *)pcb) - (CCFSZ + SPOFF));
pcb->pcb_nsaved = 0;
td->td_frame = (struct trapframe *)pcb - 1;
pcb = (struct pcb *)TLB_PHYS_TO_DIRECT(vtophys((vm_offset_t)pcb));
KASSERT(pcb > (struct pcb *)VM_MIN_DIRECT_ADDRESS,("pcb is NULL"));
td->td_pcb = pcb;
}
void *
uma_small_alloc(uma_zone_t zone, vm_size_t bytes, u_int8_t *flags, int wait)
{
vm_paddr_t pa;
vm_page_t m;
int pflags;
void *va;
PMAP_STATS_INC(uma_nsmall_alloc);
*flags = UMA_SLAB_PRIV;
pflags = malloc2vm_flags(wait) | VM_ALLOC_WIRED;
for (;;) {
m = vm_page_alloc(NULL, 0, pflags | VM_ALLOC_NOOBJ);
if (m == NULL) {
if (wait & M_NOWAIT)
return (NULL);
else
VM_WAIT;
} else
break;
}
pa = VM_PAGE_TO_PHYS(m);
if (dcache_color_ignore == 0 && m->md.color != DCACHE_COLOR(pa)) {
KASSERT(m->md.colors[0] == 0 && m->md.colors[1] == 0,
("uma_small_alloc: free page %p still has mappings!", m));
PMAP_STATS_INC(uma_nsmall_alloc_oc);
m->md.color = DCACHE_COLOR(pa);
dcache_page_inval(pa);
}
va = (void *)TLB_PHYS_TO_DIRECT(pa);
if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0)
cpu_block_zero(va, PAGE_SIZE);
return (va);
}
/*
* Implement uiomove(9) from physical memory using a combination
* of the direct mapping and sf_bufs to reduce the creation and
* destruction of ephemeral mappings.
*/
int
uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
{
struct sf_buf *sf;
struct thread *td = curthread;
struct iovec *iov;
void *cp;
vm_offset_t page_offset;
vm_paddr_t pa;
vm_page_t m;
size_t cnt;
int error = 0;
int save = 0;
KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE,
("uiomove_fromphys: mode"));
KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == curthread,
("uiomove_fromphys proc"));
save = td->td_pflags & TDP_DEADLKTREAT;
td->td_pflags |= TDP_DEADLKTREAT;
while (n > 0 && uio->uio_resid) {
iov = uio->uio_iov;
cnt = iov->iov_len;
if (cnt == 0) {
uio->uio_iov++;
uio->uio_iovcnt--;
continue;
}
if (cnt > n)
cnt = n;
page_offset = offset & PAGE_MASK;
cnt = ulmin(cnt, PAGE_SIZE - page_offset);
m = ma[offset >> PAGE_SHIFT];
pa = VM_PAGE_TO_PHYS(m);
if (m->md.color != DCACHE_COLOR(pa)) {
sf = sf_buf_alloc(m, 0);
cp = (char *)sf_buf_kva(sf) + page_offset;
} else {
sf = NULL;
cp = (char *)TLB_PHYS_TO_DIRECT(pa) + page_offset;
}
switch (uio->uio_segflg) {
case UIO_USERSPACE:
if (ticks - PCPU_GET(switchticks) >= hogticks)
uio_yield();
if (uio->uio_rw == UIO_READ)
error = copyout(cp, iov->iov_base, cnt);
else
error = copyin(iov->iov_base, cp, cnt);
if (error) {
if (sf != NULL)
sf_buf_free(sf);
goto out;
}
break;
case UIO_SYSSPACE:
if (uio->uio_rw == UIO_READ)
bcopy(cp, iov->iov_base, cnt);
else
bcopy(iov->iov_base, cp, cnt);
break;
case UIO_NOCOPY:
break;
}
if (sf != NULL)
sf_buf_free(sf);
iov->iov_base = (char *)iov->iov_base + cnt;
iov->iov_len -= cnt;
uio->uio_resid -= cnt;
uio->uio_offset += cnt;
offset += cnt;
n -= cnt;
}
out:
if (save == 0)
td->td_pflags &= ~TDP_DEADLKTREAT;
return (error);
}
/*
* Finish a fork operation, with process p2 nearly set up.
* Copy and update the pcb, set up the stack so that the child
* ready to run and return to user mode.
*/
void
cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
{
struct trapframe *tf;
struct frame *fp;
struct pcb *pcb1;
struct pcb *pcb2, *pcb2orig;
vm_offset_t sp;
int error;
int i;
KASSERT(td1 == curthread || td1 == &thread0,
("cpu_fork: p1 not curproc and not proc0"));
if ((flags & RFPROC) == 0)
return;
p2->p_md.md_sigtramp = td1->td_proc->p_md.md_sigtramp;
p2->p_md.md_utrap = utrap_hold(td1->td_proc->p_md.md_utrap);
/* The pcb must be aligned on a 64-byte boundary. */
pcb1 = td1->td_pcb;
pcb2orig = (struct pcb *)((td2->td_kstack + td2->td_kstack_pages *
PAGE_SIZE - sizeof(struct pcb)) & ~0x3fUL);
pcb2 = (struct pcb *)TLB_PHYS_TO_DIRECT(vtophys((vm_offset_t)pcb2orig));
td2->td_pcb = pcb2;
/*
* Ensure that p1's pcb is up to date.
*/
critical_enter();
if ((td1->td_frame->tf_fprs & FPRS_FEF) != 0)
savefpctx(pcb1->pcb_ufp);
critical_exit();
/* Make sure the copied windows are spilled. */
flushw();
/* Copy the pcb (this will copy the windows saved in the pcb, too). */
bcopy(pcb1, pcb2, sizeof(*pcb1));
/*
* If we're creating a new user process and we're sharing the address
* space, the parent's top most frame must be saved in the pcb. The
* child will pop the frame when it returns to user mode, and may
* overwrite it with its own data causing much suffering for the
* parent. We check if its already in the pcb, and if not copy it
* in. Its unlikely that the copyin will fail, but if so there's not
* much we can do. The parent will likely crash soon anyway in that
* case.
*/
if ((flags & RFMEM) != 0 && td1 != &thread0) {
sp = td1->td_frame->tf_sp;
for (i = 0; i < pcb1->pcb_nsaved; i++) {
if (pcb1->pcb_rwsp[i] == sp)
break;
}
if (i == pcb1->pcb_nsaved) {
error = copyin((caddr_t)sp + SPOFF, &pcb1->pcb_rw[i],
sizeof(struct rwindow));
if (error == 0) {
pcb1->pcb_rwsp[i] = sp;
pcb1->pcb_nsaved++;
}
}
}
/*
* Create a new fresh stack for the new process.
* Copy the trap frame for the return to user mode as if from a
* syscall. This copies most of the user mode register values.
*/
tf = (struct trapframe *)pcb2orig - 1;
bcopy(td1->td_frame, tf, sizeof(*tf));
tf->tf_out[0] = 0; /* Child returns zero */
tf->tf_out[1] = 0;
tf->tf_tstate &= ~TSTATE_XCC_C; /* success */
tf->tf_fprs = 0;
tf->tf_wstate = WSTATE_U64;
td2->td_frame = tf;
fp = (struct frame *)tf - 1;
fp->fr_local[0] = (u_long)fork_return;
fp->fr_local[1] = (u_long)td2;
fp->fr_local[2] = (u_long)tf;
/* Terminate stack traces at this frame. */
fp->fr_pc = fp->fr_fp = 0;
pcb2->pcb_sp = (u_long)fp - SPOFF;
pcb2->pcb_pc = (u_long)fork_trampoline - 8;
pcb2->pcb_kstack = (uint64_t)(((char *)pcb2orig) - (CCFSZ + SPOFF));
/* Setup to release spin count in fork_exit(). */
td2->td_md.md_spinlock_count = 1;
td2->td_md.md_saved_pil = 0;
/*
* Now, cpu_switch() can schedule the new process.
*/
}
/* ARGSUSED */
int
memrw(struct cdev *dev, struct uio *uio, int flags)
{
struct iovec *iov;
vm_offset_t eva;
vm_offset_t off;
vm_offset_t ova;
vm_offset_t va;
vm_prot_t prot;
vm_paddr_t pa;
vm_size_t cnt;
vm_page_t m;
int error;
int i;
uint32_t colors;
cnt = 0;
colors = 1;
error = 0;
ova = 0;
GIANT_REQUIRED;
while (uio->uio_resid > 0 && error == 0) {
iov = uio->uio_iov;
if (iov->iov_len == 0) {
uio->uio_iov++;
uio->uio_iovcnt--;
if (uio->uio_iovcnt < 0)
panic("memrw");
continue;
}
if (dev2unit(dev) == CDEV_MINOR_MEM) {
pa = uio->uio_offset & ~PAGE_MASK;
if (!is_physical_memory(pa)) {
error = EFAULT;
break;
}
off = uio->uio_offset & PAGE_MASK;
cnt = PAGE_SIZE - ((vm_offset_t)iov->iov_base &
PAGE_MASK);
cnt = ulmin(cnt, PAGE_SIZE - off);
cnt = ulmin(cnt, iov->iov_len);
m = NULL;
for (i = 0; phys_avail[i] != 0; i += 2) {
if (pa >= phys_avail[i] &&
pa < phys_avail[i + 1]) {
m = PHYS_TO_VM_PAGE(pa);
break;
}
}
if (m != NULL) {
if (ova == 0) {
if (dcache_color_ignore == 0)
colors = DCACHE_COLORS;
ova = kmem_alloc_wait(kernel_map,
PAGE_SIZE * colors);
}
if (colors != 1 && m->md.color != -1)
va = ova + m->md.color * PAGE_SIZE;
else
va = ova;
pmap_qenter(va, &m, 1);
error = uiomove((void *)(va + off), cnt,
uio);
pmap_qremove(va, 1);
} else {
va = TLB_PHYS_TO_DIRECT(pa);
error = uiomove((void *)(va + off), cnt,
uio);
}
break;
} else if (dev2unit(dev) == CDEV_MINOR_KMEM) {
va = trunc_page(uio->uio_offset);
eva = round_page(uio->uio_offset + iov->iov_len);
/*
* Make sure that all of the pages are currently
* resident so we don't create any zero fill pages.
*/
for (; va < eva; va += PAGE_SIZE)
if (pmap_kextract(va) == 0)
return (EFAULT);
prot = (uio->uio_rw == UIO_READ) ? VM_PROT_READ :
VM_PROT_WRITE;
va = uio->uio_offset;
if (va < VM_MIN_DIRECT_ADDRESS &&
kernacc((void *)va, iov->iov_len, prot) == FALSE)
return (EFAULT);
error = uiomove((void *)va, iov->iov_len, uio);
break;
}
/* else panic! */
}
if (ova != 0)
kmem_free_wakeup(kernel_map, ova, PAGE_SIZE * colors);
return (error);
}
