/*
* pick up tick count scaled to reference tick count
*/
u_int
cc_get_timecount(struct timecounter *tc)
{
struct cpu_info *ci;
int64_t rcc, cc, ncsw;
u_int gen;
retry:
ncsw = curlwp->l_ncsw;
__insn_barrier();
ci = curcpu();
if (ci->ci_cc.cc_denom == 0) {
/*
* This is our first time here on this CPU. Just
* start with reasonable initial values.
*/
ci->ci_cc.cc_cc = cpu_counter32();
ci->ci_cc.cc_val = 0;
if (ci->ci_cc.cc_gen == 0)
ci->ci_cc.cc_gen++;
ci->ci_cc.cc_denom = cpu_frequency(ci);
if (ci->ci_cc.cc_denom == 0)
ci->ci_cc.cc_denom = cc_timecounter.tc_frequency;
ci->ci_cc.cc_delta = ci->ci_cc.cc_denom;
}
/*
* read counter and re-read when the re-calibration
* strikes inbetween
*/
do {
/* pick up current generation number */
gen = ci->ci_cc.cc_gen;
/* determine local delta ticks */
cc = cpu_counter32() - ci->ci_cc.cc_cc;
if (cc < 0)
cc += 0x100000000LL;
/* scale to primary */
rcc = (cc * ci->ci_cc.cc_delta) / ci->ci_cc.cc_denom
+ ci->ci_cc.cc_val;
} while (gen == 0 || gen != ci->ci_cc.cc_gen);
__insn_barrier();
if (ncsw != curlwp->l_ncsw) {
/* Was preempted */
goto retry;
}
return rcc;
}
/*
* Reads a consistent set of time-base values from Xen, into a shadow data
* area. Must be called at splclock.
*/
static void
get_time_values_from_xen(void)
{
do {
shadow_time_version = HYPERVISOR_shared_info->time_version2;
__insn_barrier();
shadow_tv.tv_sec = HYPERVISOR_shared_info->wc_sec;
shadow_tv.tv_usec = HYPERVISOR_shared_info->wc_usec;
shadow_tsc_stamp = HYPERVISOR_shared_info->tsc_timestamp <<
HYPERVISOR_shared_info->rdtsc_bitshift;
shadow_system_time = HYPERVISOR_shared_info->system_time;
__insn_barrier();
} while (shadow_time_version != HYPERVISOR_shared_info->time_version1);
}
static void
init_interface(void)
{
block_io_op_t op;
reset_interface();
if (blk_ring == NULL) {
op.cmd = BLOCK_IO_OP_RING_ADDRESS;
(void)HYPERVISOR_block_io_op(&op);
blk_ring = (blk_ring_t *)uvm_km_valloc_align(kernel_map,
PAGE_SIZE, PAGE_SIZE);
pmap_kenter_ma((vaddr_t)blk_ring, op.u.ring_mfn << PAGE_SHIFT,
VM_PROT_READ|VM_PROT_WRITE);
DPRINTF(XBDB_SETUP, ("init_interface: "
"ring va %p and wired to %p\n",
blk_ring, (void *)(op.u.ring_mfn << PAGE_SHIFT)));
blk_ring->req_prod = blk_ring->resp_prod =
resp_cons = req_prod = last_req_prod = 0;
event_set_handler(_EVENT_BLKDEV, &xbd_response_handler,
NULL, IPL_BIO);
hypervisor_enable_event(_EVENT_BLKDEV);
}
__insn_barrier();
state = STATE_ACTIVE;
}
void smp_sched_handler(void)
{
unsigned flgs;
unsigned cpu = cpuid;
flgs = sched_ipi_data[cpu].flags;
if (flgs) {
struct proc * p;
p = (struct proc *)sched_ipi_data[cpu].data;
if (flgs & SCHED_IPI_STOP_PROC) {
RTS_SET(p, RTS_PROC_STOP);
}
if (flgs & SCHED_IPI_SAVE_CTX) {
/* all context has been saved already, FPU remains */
if (proc_used_fpu(p) &&
get_cpulocal_var(fpu_owner) == p) {
disable_fpu_exception();
save_local_fpu(p, FALSE /*retain*/);
/* we're preparing to migrate somewhere else */
release_fpu(p);
}
}
if (flgs & SCHED_IPI_VM_INHIBIT) {
RTS_SET(p, RTS_VMINHIBIT);
}
}
__insn_barrier();
sched_ipi_data[cpu].flags = 0;
}
static void udelay(unsigned int usec)
{
struct at91tctmr_softc *sc = at91tctmr_sc;
u_int32_t prev_cvr, cvr, divi = READ_TC(sc, TC_RC), diff;
int prev_ticks, ticks, ticks2;
unsigned footick = (sc->sc_timerclock * 64ULL / 1000000UL);
if (usec > 0) {
prev_ticks = hardclock_ticks;
__insn_barrier();
prev_cvr = READ_TC(sc, TC_CV);
ticks = hardclock_ticks;
__insn_barrier();
if (ticks != prev_ticks) {
prev_cvr = READ_TC(sc, TC_CV);
prev_ticks = ticks;
}
for (;;) {
ticks = hardclock_ticks;
__insn_barrier();
cvr = READ_TC(sc, TC_CV);
ticks2 = hardclock_ticks;
__insn_barrier();
if (ticks2 != ticks) {
cvr = READ_TC(sc, TC_CV);
}
diff = (ticks2 - prev_ticks) * divi;
if (cvr < prev_cvr) {
if (!diff)
diff = divi;
diff -= prev_cvr - cvr;
} else
diff += cvr - prev_cvr;
diff = diff * 64 / footick;
if (diff) {
if (usec <= diff)
break;
prev_ticks = ticks2;
prev_cvr = (prev_cvr + footick * diff / 64) % divi;
usec -= diff;
}
}
}
}
void
delay(int ms)
{
/*
* XXX need *real* clock calibration.
*/
volatile register int N = ms * DELAY_CALIBRATE;
for (; --N;)
__insn_barrier();
}
int
virtio_to_queue(struct virtio_device *dev, int qidx, struct vumap_phys *bufs,
size_t num, void *data)
{
u16_t free_first;
int left;
struct virtio_queue *q = &dev->queues[qidx];
struct vring *vring = &q->vring;
assert(0 <= qidx && qidx <= dev->num_queues);
if (!data)
panic("%s: NULL data received queue %d", dev->name, qidx);
free_first = q->free_head;
left = (int)q->free_num - (int)num;
if (left < dev->threads)
set_indirect_descriptors(dev, q, bufs, num);
else
set_direct_descriptors(q, bufs, num);
/* Next index for host is old free_head */
vring->avail->ring[vring->avail->idx % q->num] = free_first;
/* Provided by the caller to identify this slot */
q->data[free_first] = data;
/* Make sure the host sees the new descriptors */
__insn_barrier();
/* advance last idx */
vring->avail->idx += 1;
/* Make sure the host sees the avail->idx */
__insn_barrier();
/* kick it! */
kick_queue(dev, qidx);
return 0;
}
/*
* Add a mask to cpl, and return the old value of cpl.
*/
int
splraise(int nlevel)
{
int olevel;
struct cpu_info *ci = curcpu();
olevel = ci->ci_ilevel;
if (nlevel > olevel)
ci->ci_ilevel = nlevel;
__insn_barrier();
return (olevel);
}
void
splx(int ncpl)
{
struct cpu_info *ci = curcpu();
__insn_barrier();
__asm volatile("sync; eieio"); /* reorder protect */
ci->ci_cpl = ncpl;
if (have_pending_intr_p(ci, ncpl))
pic_do_pending_int();
__asm volatile("sync; eieio"); /* reorder protect */
}
int
spllower(int ncpl)
{
struct cpu_info *ci = curcpu();
int ocpl;
__insn_barrier();
__asm volatile("sync; eieio"); /* reorder protect */
ocpl = ci->ci_cpl;
ci->ci_cpl = ncpl;
if (have_pending_intr_p(ci, ncpl))
pic_do_pending_int();
__asm volatile("sync; eieio"); /* reorder protect */
return ocpl;
}
int
splraise(int ncpl)
{
struct cpu_info *ci = curcpu();
int ocpl;
if (ncpl == ci->ci_cpl) return ncpl;
__asm volatile("sync; eieio"); /* don't reorder.... */
ocpl = ci->ci_cpl;
KASSERT(ncpl < NIPL);
ci->ci_cpl = max(ncpl, ocpl);
__asm volatile("sync; eieio"); /* reorder protect */
__insn_barrier();
return ocpl;
}
static void
linux_work_unlock(struct work_struct *work)
{
struct cpu_info *ci;
int s;
__cpu_simple_unlock(&work->w_lock);
/* XXX Copypasta of MUTEX_SPIN_SPLRESTORE. */
ci = curcpu();
s = ci->ci_mtx_oldspl;
__insn_barrier();
if (++ci->ci_mtx_count == 0)
splx(s);
}
static void
linux_work_lock(struct work_struct *work)
{
struct cpu_info *ci;
int cnt, s;
/* XXX Copypasta of MUTEX_SPIN_SPLRAISE. */
s = splvm();
ci = curcpu();
cnt = ci->ci_mtx_count--;
__insn_barrier();
if (cnt == 0)
ci->ci_mtx_oldspl = s;
__cpu_simple_lock(&work->w_lock);
}
/*
* tell another cpu about a task to do and return only after the cpu acks that
* the task is finished. Also wait before it finishes task sent by another cpu
* to the same one.
*/
static void smp_schedule_sync(struct proc * p, unsigned task)
{
unsigned cpu = p->p_cpu;
unsigned mycpu = cpuid;
assert(cpu != mycpu);
/*
* if some other cpu made a request to the same cpu, wait until it is
* done before proceeding
*/
if (sched_ipi_data[cpu].flags != 0) {
BKL_UNLOCK();
while (sched_ipi_data[cpu].flags != 0) {
if (sched_ipi_data[mycpu].flags) {
BKL_LOCK();
smp_sched_handler();
BKL_UNLOCK();
}
}
BKL_LOCK();
}
sched_ipi_data[cpu].data = (u32_t) p;
sched_ipi_data[cpu].flags |= task;
__insn_barrier();
arch_send_smp_schedule_ipi(cpu);
/* wait until the destination cpu finishes its job */
BKL_UNLOCK();
while (sched_ipi_data[cpu].flags != 0) {
if (sched_ipi_data[mycpu].flags) {
BKL_LOCK();
smp_sched_handler();
BKL_UNLOCK();
}
}
BKL_LOCK();
}
/*
* Restore a value to cpl (unmasking interrupts). If any unmasked
* interrupts are pending, call Xspllower() to process them.
*/
void
spllower(int nlevel)
{
struct cpu_info *ci = curcpu();
uint32_t imask;
u_long psl;
if (ci->ci_ilevel <= nlevel)
return;
__insn_barrier();
imask = IUNMASK(ci, nlevel);
psl = x86_read_psl();
x86_disable_intr();
if (ci->ci_ipending & imask) {
KASSERT(psl == 0);
Xspllower(nlevel);
/* Xspllower does enable_intr() */
} else {
ci->ci_ilevel = nlevel;
x86_write_psl(psl);
}
}
/*===========================================================================*
* atl2_writev *
*===========================================================================*/
static void atl2_writev(const message *m, int from_int)
{
/* Write packet data.
*/
iovec_s_t *iovp;
size_t off, count, left, pos, skip;
vir_bytes size;
u8_t *sizep;
int i, j, r, batch, maxnum;
/* We can deal with only one write request from Inet at a time. */
assert(from_int || !(state.flags & ATL2_FLAG_WRITE_PEND));
state.task_endpt = m->m_source;
/* If we are already certain that the packet won't fit, bail out.
* Keep at least some space between TxD head and tail, as it is not
* clear whether the device deals well with the case that they collide.
*/
if (state.txs_num >= ATL2_TXS_COUNT)
goto suspend;
maxnum = ATL2_TXD_BUFSIZE - ETH_MIN_PACK_SIZE - sizeof(u32_t);
if (state.txd_num >= maxnum)
goto suspend;
/* Optimistically try to copy in the data; suspend if it turns out
* that it does not fit.
*/
off = 0;
count = 0;
left = state.txd_num - sizeof(u32_t);
pos = (state.txd_tail + state.txd_num +
sizeof(u32_t)) % ATL2_TXD_BUFSIZE;
for (i = 0; i < m->DL_COUNT; i += batch) {
/* Copy in the next batch. */
batch = MIN(m->DL_COUNT - i, NR_IOREQS);
r = sys_safecopyfrom(m->m_source, m->DL_GRANT, off,
(vir_bytes) iovec, batch * sizeof(iovec[0]));
if (r != OK)
panic("vector copy failed: %d", r);
/* Copy in each element in the batch. */
for (j = 0, iovp = iovec; j < batch; j++, iovp++) {
size = iovp->iov_size;
if (size > left)
goto suspend;
skip = 0;
if (size > ATL2_TXD_BUFSIZE - pos) {
skip = ATL2_TXD_BUFSIZE - pos;
r = sys_safecopyfrom(m->m_source,
iovp->iov_grant, 0,
(vir_bytes) (state.txd_base + pos),
skip);
if (r != OK)
panic("safe copy failed: %d", r);
pos = 0;
}
r = sys_safecopyfrom(m->m_source, iovp->iov_grant,
skip, (vir_bytes) (state.txd_base + pos),
size - skip);
if (r != OK)
panic("safe copy failed: %d", r);
pos = (pos + size - skip) % ATL2_TXD_BUFSIZE;
left -= size;
count += size;
}
off += batch * sizeof(iovec[0]);
}
assert(count <= ETH_MAX_PACK_SIZE_TAGGED);
/* Write the length to the DWORD right before the packet. */
sizep = state.txd_base +
(state.txd_tail + state.txd_num) % ATL2_TXD_BUFSIZE;
* (u32_t *) sizep = count;
/* Update the TxD head. */
state.txd_num += sizeof(u32_t) + ATL2_ALIGN_32(count);
pos = ATL2_ALIGN_32(pos) % ATL2_TXD_BUFSIZE;
assert((int) pos ==
(state.txd_tail + state.txd_num) % ATL2_TXD_BUFSIZE);
/* Initialize and update the TxS head. */
state.txs_base[(state.txs_tail + state.txs_num) % ATL2_TXS_COUNT] = 0;
state.txs_num++;
/* Tell the device about our new position. */
__insn_barrier();
ATL2_WRITE_U32(ATL2_TXD_IDX_REG, pos / sizeof(u32_t));
/* We have now successfully set up the transmission of a packet. */
state.flags &= ~ATL2_FLAG_WRITE_PEND;
state.flags |= ATL2_FLAG_PACK_SENT;
/* If called from the interrupt handler, the caller will reply. */
if (!from_int)
atl2_reply();
return;
suspend:
/* We cannot transmit the packet at this time. If we were not already
* trying to resume transmission, save the write request for later,
* and tell Inet that the request has been suspended.
*/
if (from_int)
return;
state.flags |= ATL2_FLAG_WRITE_PEND;
state.write_msg = *m;
atl2_reply();
}
int
virtio_from_queue(struct virtio_device *dev, int qidx, void **data,
size_t *len)
{
struct virtio_queue *q;
struct vring *vring;
struct vring_used_elem *uel;
struct vring_desc *vd;
int count = 0;
u16_t idx;
u16_t used_idx;
assert(0 <= qidx && qidx < dev->num_queues);
q = &dev->queues[qidx];
vring = &q->vring;
/* Make sure we see changes done by the host */
__insn_barrier();
/* The index from the host */
used_idx = vring->used->idx % q->num;
/* We already saw this one, nothing to do here */
if (q->last_used == used_idx)
return -1;
/* Get the vring_used element */
uel = &q->vring.used->ring[q->last_used];
/* Update the last used element */
q->last_used = (q->last_used + 1) % q->num;
/* index of the used element */
idx = uel->id % q->num;
assert(q->data[idx] != NULL);
/* Get the descriptor */
vd = &vring->desc[idx];
/* Unconditionally set the tail->next to the first used one */
assert(vring->desc[q->free_tail].flags & VRING_DESC_F_NEXT);
vring->desc[q->free_tail].next = idx;
/* Find the last index, eventually there has to be one
* without a the next flag.
*
* FIXME: Protect from endless loop
*/
while (vd->flags & VRING_DESC_F_NEXT) {
if (vd->flags & VRING_DESC_F_INDIRECT)
clear_indirect_table(dev, vd);
idx = vd->next;
vd = &vring->desc[idx];
count++;
}
/* Didn't count the last one */
count++;
if (vd->flags & VRING_DESC_F_INDIRECT)
clear_indirect_table(dev, vd);
/* idx points to the tail now, update the queue */
q->free_tail = idx;
assert(!(vd->flags & VRING_DESC_F_NEXT));
/* We can always connect the tail with the head */
vring->desc[q->free_tail].next = q->free_head;
vring->desc[q->free_tail].flags = VRING_DESC_F_NEXT;
q->free_num += count;
assert(q->free_num <= q->num);
*data = q->data[uel->id];
q->data[uel->id] = NULL;
if (len != NULL)
*len = uel->len;
return 0;
}
/* _mcount; may be static, inline, etc */
_MCOUNT_DECL(u_long frompc, u_long selfpc)
{
u_short *frompcindex;
struct tostruct *top, *prevtop;
struct gmonparam *p;
long toindex;
#if defined(_KERNEL) && !defined(_RUMPKERNEL)
int s;
#endif
#if defined(_REENTRANT) && !defined(_KERNEL)
if (__isthreaded) {
/* prevent re-entry via thr_getspecific */
if (_gmonparam.state != GMON_PROF_ON)
return;
_gmonparam.state = GMON_PROF_BUSY;
p = thr_getspecific(_gmonkey);
if (p == NULL) {
/* Prevent recursive calls while allocating */
thr_setspecific(_gmonkey, &_gmondummy);
p = _m_gmon_alloc();
}
_gmonparam.state = GMON_PROF_ON;
} else
#endif
p = &_gmonparam;
/*
* check that we are profiling
* and that we aren't recursively invoked.
*/
if (p->state != GMON_PROF_ON)
return;
#if defined(_KERNEL) && !defined(_RUMPKERNEL)
MCOUNT_ENTER;
#ifdef MULTIPROCESSOR
__cpu_simple_lock(&__mcount_lock);
__insn_barrier();
#endif
#endif
p->state = GMON_PROF_BUSY;
/*
* check that frompcindex is a reasonable pc value.
* for example: signal catchers get called from the stack,
* not from text space. too bad.
*/
frompc -= p->lowpc;
if (frompc > p->textsize)
goto done;
#if (HASHFRACTION & (HASHFRACTION - 1)) == 0
if (p->hashfraction == HASHFRACTION)
frompcindex =
&p->froms[
(size_t)(frompc / (HASHFRACTION * sizeof(*p->froms)))];
else
#endif
frompcindex =
&p->froms[
(size_t)(frompc / (p->hashfraction * sizeof(*p->froms)))];
toindex = *frompcindex;
if (toindex == 0) {
/*
* first time traversing this arc
*/
toindex = ++p->tos[0].link;
if (toindex >= p->tolimit)
/* halt further profiling */
goto overflow;
*frompcindex = (u_short)toindex;
top = &p->tos[(size_t)toindex];
top->selfpc = selfpc;
top->count = 1;
top->link = 0;
goto done;
}
top = &p->tos[(size_t)toindex];
if (top->selfpc == selfpc) {
/*
* arc at front of chain; usual case.
*/
top->count++;
goto done;
}
/*
* have to go looking down chain for it.
* top points to what we are looking at,
* prevtop points to previous top.
* we know it is not at the head of the chain.
*/
for (; /* goto done */; ) {
if (top->link == 0) {
/*
* top is end of the chain and none of the chain
* had top->selfpc == selfpc.
* so we allocate a new tostruct
* and link it to the head of the chain.
*/
toindex = ++p->tos[0].link;
if (toindex >= p->tolimit)
goto overflow;
top = &p->tos[(size_t)toindex];
top->selfpc = selfpc;
top->count = 1;
top->link = *frompcindex;
*frompcindex = (u_short)toindex;
goto done;
}
/*
* otherwise, check the next arc on the chain.
*/
prevtop = top;
top = &p->tos[top->link];
if (top->selfpc == selfpc) {
/*
* there it is.
* increment its count
* move it to the head of the chain.
*/
top->count++;
toindex = prevtop->link;
prevtop->link = top->link;
top->link = *frompcindex;
*frompcindex = (u_short)toindex;
goto done;
}
}
done:
p->state = GMON_PROF_ON;
#if defined(_KERNEL) && !defined(_RUMPKERNEL)
#ifdef MULTIPROCESSOR
__insn_barrier();
__cpu_simple_unlock(&__mcount_lock);
#endif
MCOUNT_EXIT;
#endif
return;
overflow:
p->state = GMON_PROF_ERROR;
#if defined(_KERNEL) && !defined(_RUMPKERNEL)
#ifdef MULTIPROCESSOR
__insn_barrier();
__cpu_simple_unlock(&__mcount_lock);
#endif
MCOUNT_EXIT;
#endif
return;
}
/*===========================================================================*
* atl2_rx_advance *
*===========================================================================*/
static void atl2_rx_advance(int next)
{
/* Advance the RxD tail by as many failed receipts as possible, and
* see if there is an actual packet left to receive. If 'next' is set,
* the packet at the current tail has been processed.
*/
int update_tail;
rxd_t *rxd;
u32_t hdr, size;
update_tail = FALSE;
if (next) {
state.rxd_tail = (state.rxd_tail + 1) % ATL2_RXD_COUNT;
update_tail = TRUE;
ATL2_DEBUG(("ATL2: successfully received packet\n"));
state.flags &= ~ATL2_FLAG_RX_AVAIL;
}
assert(!(state.flags & ATL2_FLAG_RX_AVAIL));
for (;;) {
/* Check the RxD tail for updates. */
rxd = &state.rxd_base[state.rxd_tail];
hdr = rxd->hdr;
if (!(hdr & ATL2_RXD_UPDATE))
break;
rxd->hdr = hdr & ~(ATL2_RXD_UPDATE);
/* Update statistics. */
atl2_rx_stat(hdr);
/* Stop at the first successful receipt. The packet will be
* picked up by Inet later.
*/
size = hdr & ATL2_RXD_SIZE_MASK;
if ((hdr & ATL2_RXD_SUCCESS) && size >= ETH_MIN_PACK_SIZE) {
ATL2_DEBUG(("ATL2: packet available, size %ld\n",
size));
state.flags |= ATL2_FLAG_RX_AVAIL;
break;
}
ATL2_DEBUG(("ATL2: packet receipt failed\n"));
/* Advance tail. */
state.rxd_tail = (state.rxd_tail + 1) % ATL2_RXD_COUNT;
update_tail = TRUE;
}
/* If new RxD descriptors are now up for reuse, tell the device. */
if (update_tail) {
__insn_barrier();
ATL2_WRITE_U32(ATL2_RXD_IDX_REG, state.rxd_tail);
}
}
