int evtchn_unmask(unsigned int port) { struct domain *d = current->domain; struct vcpu *v; ASSERT(spin_is_locked(&d->event_lock)); if ( unlikely(!port_is_valid(d, port)) ) return -EINVAL; v = d->vcpu[evtchn_from_port(d, port)->notify_vcpu_id]; /* * These operations must happen in strict order. Based on * include/xen/event.h:evtchn_set_pending(). */ if ( test_and_clear_bit(port, &shared_info(d, evtchn_mask)) && test_bit (port, &shared_info(d, evtchn_pending)) && !test_and_set_bit (port / BITS_PER_EVTCHN_WORD(d), &vcpu_info(v, evtchn_pending_sel)) ) { vcpu_mark_events_pending(v); } return 0; }
static bool evtchn_2l_is_masked(const struct domain *d, evtchn_port_t port) { unsigned int max_ports = BITS_PER_EVTCHN_WORD(d) * BITS_PER_EVTCHN_WORD(d); ASSERT(port < max_ports); return port >= max_ports || test_bit(port, &shared_info(d, evtchn_mask)); }
static void evtchn_2l_unmask(struct domain *d, struct evtchn *evtchn) { struct vcpu *v = d->vcpu[evtchn->notify_vcpu_id]; unsigned int port = evtchn->port; /* * These operations must happen in strict order. Based on * evtchn_2l_set_pending() above. */ if ( test_and_clear_bit(port, &shared_info(d, evtchn_mask)) && test_bit (port, &shared_info(d, evtchn_pending)) && !test_and_set_bit (port / BITS_PER_EVTCHN_WORD(d), &vcpu_info(v, evtchn_pending_sel)) ) { vcpu_mark_events_pending(v); } }
void update_domain_wallclock_time(struct domain *d) { uint32_t *wc_version; uint64_t sec; spin_lock(&wc_lock); wc_version = &shared_info(d, wc_version); *wc_version = version_update_begin(*wc_version); smp_wmb(); sec = wc_sec + d->time_offset_seconds; shared_info(d, wc_sec) = sec; shared_info(d, wc_nsec) = wc_nsec; #ifdef CONFIG_X86 if ( likely(!has_32bit_shinfo(d)) ) d->shared_info->native.wc_sec_hi = sec >> 32; else
static int evtchn_set_pending(struct vcpu *v, int port) { struct domain *d = v->domain; int vcpuid; /* * The following bit operations must happen in strict order. * NB. On x86, the atomic bit operations also act as memory barriers. * There is therefore sufficiently strict ordering for this architecture -- * others may require explicit memory barriers. */ if ( test_and_set_bit(port, &shared_info(d, evtchn_pending)) ) return 1; if ( !test_bit (port, &shared_info(d, evtchn_mask)) && !test_and_set_bit(port / BITS_PER_EVTCHN_WORD(d), &vcpu_info(v, evtchn_pending_sel)) ) { vcpu_mark_events_pending(v); } /* Check if some VCPU might be polling for this event. */ if ( likely(bitmap_empty(d->poll_mask, d->max_vcpus)) ) return 0; /* Wake any interested (or potentially interested) pollers. */ for ( vcpuid = find_first_bit(d->poll_mask, d->max_vcpus); vcpuid < d->max_vcpus; vcpuid = find_next_bit(d->poll_mask, d->max_vcpus, vcpuid+1) ) { v = d->vcpu[vcpuid]; if ( ((v->poll_evtchn <= 0) || (v->poll_evtchn == port)) && test_and_clear_bit(vcpuid, d->poll_mask) ) { v->poll_evtchn = 0; vcpu_unblock(v); } } return 0; }
static void evtchn_2l_set_pending(struct vcpu *v, struct evtchn *evtchn) { struct domain *d = v->domain; unsigned int port = evtchn->port; /* * The following bit operations must happen in strict order. * NB. On x86, the atomic bit operations also act as memory barriers. * There is therefore sufficiently strict ordering for this architecture -- * others may require explicit memory barriers. */ if ( test_and_set_bit(port, &shared_info(d, evtchn_pending)) ) return; if ( !test_bit (port, &shared_info(d, evtchn_mask)) && !test_and_set_bit(port / BITS_PER_EVTCHN_WORD(d), &vcpu_info(v, evtchn_pending_sel)) ) { vcpu_mark_events_pending(v); } evtchn_check_pollers(d, port); }
struct vcpu *alloc_vcpu( struct domain *d, unsigned int vcpu_id, unsigned int cpu_id) { struct vcpu *v; BUG_ON((!is_idle_domain(d) || vcpu_id) && d->vcpu[vcpu_id]); if ( (v = alloc_vcpu_struct()) == NULL ) return NULL; v->domain = d; v->vcpu_id = vcpu_id; spin_lock_init(&v->virq_lock); tasklet_init(&v->continue_hypercall_tasklet, NULL, 0); if ( !zalloc_cpumask_var(&v->cpu_affinity) || !zalloc_cpumask_var(&v->cpu_affinity_tmp) || !zalloc_cpumask_var(&v->cpu_affinity_saved) || !zalloc_cpumask_var(&v->vcpu_dirty_cpumask) ) goto fail_free; if ( is_idle_domain(d) ) { v->runstate.state = RUNSTATE_running; } else { v->runstate.state = RUNSTATE_offline; v->runstate.state_entry_time = NOW(); set_bit(_VPF_down, &v->pause_flags); v->vcpu_info = ((vcpu_id < XEN_LEGACY_MAX_VCPUS) ? (vcpu_info_t *)&shared_info(d, vcpu_info[vcpu_id]) : &dummy_vcpu_info); v->vcpu_info_mfn = INVALID_MFN; init_waitqueue_vcpu(v); } if ( sched_init_vcpu(v, cpu_id) != 0 ) goto fail_wq; if ( vcpu_initialise(v) != 0 ) { sched_destroy_vcpu(v); fail_wq: destroy_waitqueue_vcpu(v); fail_free: free_cpumask_var(v->cpu_affinity); free_cpumask_var(v->cpu_affinity_tmp); free_cpumask_var(v->cpu_affinity_saved); free_cpumask_var(v->vcpu_dirty_cpumask); free_vcpu_struct(v); return NULL; } d->vcpu[vcpu_id] = v; if ( vcpu_id != 0 ) { int prev_id = v->vcpu_id - 1; while ( (prev_id >= 0) && (d->vcpu[prev_id] == NULL) ) prev_id--; BUG_ON(prev_id < 0); v->next_in_list = d->vcpu[prev_id]->next_in_list; d->vcpu[prev_id]->next_in_list = v; } /* Must be called after making new vcpu visible to for_each_vcpu(). */ vcpu_check_shutdown(v); domain_update_node_affinity(d); return v; }
static long __evtchn_close(struct domain *d1, int port1) { struct domain *d2 = NULL; struct vcpu *v; struct evtchn *chn1, *chn2; int port2; long rc = 0; again: spin_lock(&d1->event_lock); if ( !port_is_valid(d1, port1) ) { rc = -EINVAL; goto out; } chn1 = evtchn_from_port(d1, port1); /* Guest cannot close a Xen-attached event channel. */ if ( unlikely(consumer_is_xen(chn1)) ) { rc = -EINVAL; goto out; } switch ( chn1->state ) { case ECS_FREE: case ECS_RESERVED: rc = -EINVAL; goto out; case ECS_UNBOUND: break; case ECS_PIRQ: { struct pirq *pirq = pirq_info(d1, chn1->u.pirq.irq); if ( !pirq ) break; if ( !is_hvm_domain(d1) ) pirq_guest_unbind(d1, pirq); pirq->evtchn = 0; pirq_cleanup_check(pirq, d1); unlink_pirq_port(chn1, d1->vcpu[chn1->notify_vcpu_id]); #ifdef CONFIG_X86 if ( is_hvm_domain(d1) && domain_pirq_to_irq(d1, pirq->pirq) > 0 ) unmap_domain_pirq_emuirq(d1, pirq->pirq); #endif break; } case ECS_VIRQ: for_each_vcpu ( d1, v ) { if ( v->virq_to_evtchn[chn1->u.virq] != port1 ) continue; v->virq_to_evtchn[chn1->u.virq] = 0; spin_barrier(&v->virq_lock); } break; case ECS_IPI: break; case ECS_INTERDOMAIN: if ( d2 == NULL ) { d2 = chn1->u.interdomain.remote_dom; /* If we unlock d1 then we could lose d2. Must get a reference. */ if ( unlikely(!get_domain(d2)) ) BUG(); if ( d1 < d2 ) { spin_lock(&d2->event_lock); } else if ( d1 != d2 ) { spin_unlock(&d1->event_lock); spin_lock(&d2->event_lock); goto again; } } else if ( d2 != chn1->u.interdomain.remote_dom ) { /* * We can only get here if the port was closed and re-bound after * unlocking d1 but before locking d2 above. We could retry but * it is easier to return the same error as if we had seen the * port in ECS_CLOSED. It must have passed through that state for * us to end up here, so it's a valid error to return. */ rc = -EINVAL; goto out; } port2 = chn1->u.interdomain.remote_port; BUG_ON(!port_is_valid(d2, port2)); chn2 = evtchn_from_port(d2, port2); BUG_ON(chn2->state != ECS_INTERDOMAIN); BUG_ON(chn2->u.interdomain.remote_dom != d1); chn2->state = ECS_UNBOUND; chn2->u.unbound.remote_domid = d1->domain_id; break; default: BUG(); } /* Clear pending event to avoid unexpected behavior on re-bind. */ clear_bit(port1, &shared_info(d1, evtchn_pending)); /* Reset binding to vcpu0 when the channel is freed. */ chn1->state = ECS_FREE; chn1->notify_vcpu_id = 0; xsm_evtchn_close_post(chn1); out: if ( d2 != NULL ) { if ( d1 != d2 ) spin_unlock(&d2->event_lock); put_domain(d2); } spin_unlock(&d1->event_lock); return rc; }
static long do_poll(struct sched_poll *sched_poll) { struct vcpu *v = current; struct domain *d = v->domain; evtchn_port_t port; long rc; unsigned int i; /* Fairly arbitrary limit. */ if ( sched_poll->nr_ports > 128 ) return -EINVAL; if ( !guest_handle_okay(sched_poll->ports, sched_poll->nr_ports) ) return -EFAULT; set_bit(_VPF_blocked, &v->pause_flags); v->poll_evtchn = -1; set_bit(v->vcpu_id, d->poll_mask); #ifndef CONFIG_X86 /* set_bit() implies mb() on x86 */ /* Check for events /after/ setting flags: avoids wakeup waiting race. */ smp_mb(); /* * Someone may have seen we are blocked but not that we are polling, or * vice versa. We are certainly being woken, so clean up and bail. Beyond * this point others can be guaranteed to clean up for us if they wake us. */ rc = 0; if ( (v->poll_evtchn == 0) || !test_bit(_VPF_blocked, &v->pause_flags) || !test_bit(v->vcpu_id, d->poll_mask) ) goto out; #endif rc = 0; if ( local_events_need_delivery() ) goto out; for ( i = 0; i < sched_poll->nr_ports; i++ ) { rc = -EFAULT; if ( __copy_from_guest_offset(&port, sched_poll->ports, i, 1) ) goto out; rc = -EINVAL; if ( port >= MAX_EVTCHNS(d) ) goto out; rc = 0; if ( test_bit(port, &shared_info(d, evtchn_pending)) ) goto out; } if ( sched_poll->nr_ports == 1 ) v->poll_evtchn = port; if ( sched_poll->timeout != 0 ) set_timer(&v->poll_timer, sched_poll->timeout); TRACE_2D(TRC_SCHED_BLOCK, d->domain_id, v->vcpu_id); raise_softirq(SCHEDULE_SOFTIRQ); return 0; out: v->poll_evtchn = 0; clear_bit(v->vcpu_id, d->poll_mask); clear_bit(_VPF_blocked, &v->pause_flags); return rc; }
static void evtchn_2l_clear_pending(struct domain *d, struct evtchn *evtchn) { clear_bit(evtchn->port, &shared_info(d, evtchn_pending)); }
static bool_t evtchn_2l_is_masked(struct domain *d, const struct evtchn *evtchn) { return test_bit(evtchn->port, &shared_info(d, evtchn_mask)); }