int
ipe_init(void)
{
spinlock_acquire(&ipe_init_alloc_lock);
local_recv_buffer = remote_recv_buffer[lapic_id] =
(ipe_packet_t *)KADDR_DIRECT(kalloc_pages(RBUF_PAGES));
spinlock_release(&ipe_init_alloc_lock);
eproc_open(&ipe_eproc, "ipe", (void(*)(void))proc_wait_try, NULL, 8192);
event_open(ipe_event, &ipe_eproc.event_pool, do_ipe, NULL);
event_open(&ipe_timer.event, &ipe_eproc.event_pool, do_ipe, NULL);
timer_open(&ipe_timer, timer_tick + IPE_REFRESH_INV * timer_freq);
/* ALL CPU BARRIER {{{ */
/* XXX: use naive CAS here =_= for atomic inc */
while (1)
{
int old = ipe_ready;
if (cmpxchg32(&ipe_ready, old, old + 1) == old) break;
}
while (ipe_ready != lcpu_count) ;
/* }}} */
return 0;
}
void lcpu_init(void)
{
while (1) {
int old = lcpu_init_count;
if (cmpxchg32(&lcpu_init_count, old, old + 1) == old)
break;
}
jump_kern();
}
/*
* This will "rendez-vous" all threads on the core to the rendez-vous
* id "sig". You need to make sure that "sig" is different from the
* previous rendez vous. The sig value must be between 0 and 7 with
* boot time being set to 0.
*
* Note: in theory, we could just use a flip flop "sig" in the thread
* structure (binary rendez-vous with no argument). This is a bit more
* debuggable and better at handling timeouts (arguably).
*
* This should be called with the no lock held
*/
static void hmi_rendez_vous(uint32_t sig)
{
struct cpu_thread *t = this_cpu();
uint32_t my_id = cpu_get_thread_index(t);
uint32_t my_shift = my_id << 2;
uint32_t *sptr = t->core_hmi_state_ptr;
uint32_t val, prev, shift, i;
uint64_t timeout;
assert(sig <= 0x7);
/*
* Mark ourselves as having reached the rendez vous point with
* the exit bit cleared
*/
do {
val = prev = *sptr;
val &= ~(0xfu << my_shift);
val |= sig << my_shift;
} while (cmpxchg32(sptr, prev, val) != prev);
/*
* Wait for everybody else to reach that point, ignore the
* exit bit as another thread could have already set it.
*/
for (i = 0; i < cpu_thread_count; i++) {
shift = i << 2;
timeout = TIMEOUT_LOOPS;
while (((*sptr >> shift) & 0x7) != sig && --timeout)
cpu_relax();
if (!timeout)
prlog(PR_ERR, "Rendez-vous stage 1 timeout, CPU 0x%x"
" waiting for thread %d (sptr=%08x)\n",
t->pir, i, *sptr);
}
/* Set the exit bit */
do {
val = prev = *sptr;
val &= ~(0xfu << my_shift);
val |= (sig | 8) << my_shift;
} while (cmpxchg32(sptr, prev, val) != prev);
/* At this point, we need to wait for everybody else to have a value
* that is *not* sig. IE. they either have set the exit bit *or* they
* have changed the rendez-vous (meaning they have moved on to another
* rendez vous point).
*/
for (i = 0; i < cpu_thread_count; i++) {
shift = i << 2;
timeout = TIMEOUT_LOOPS;
while (((*sptr >> shift) & 0xf) == sig && --timeout)
cpu_relax();
if (!timeout)
prlog(PR_ERR, "Rendez-vous stage 2 timeout, CPU 0x%x"
" waiting for thread %d (sptr=%08x)\n",
t->pir, i, *sptr);
}
}