/* * thread_fast_set_cthread_self: Sets the machine kernel thread ID of the * current thread to the given thread ID; fast version for 32-bit processes * * Parameters: self Thread ID to set * * Returns: 0 Success * !0 Not success */ kern_return_t thread_fast_set_cthread_self(uint32_t self) { thread_t thread = current_thread(); pcb_t pcb = thread->machine.pcb; struct real_descriptor desc = { .limit_low = 1, .limit_high = 0, .base_low = self & 0xffff, .base_med = (self >> 16) & 0xff, .base_high = (self >> 24) & 0xff, .access = ACC_P|ACC_PL_U|ACC_DATA_W, .granularity = SZ_32|SZ_G, }; current_thread()->machine.pcb->cthread_self = (uint64_t) self; /* preserve old func too */ /* assign descriptor */ mp_disable_preemption(); pcb->cthread_desc = desc; *ldt_desc_p(USER_CTHREAD) = desc; saved_state32(pcb->iss)->gs = USER_CTHREAD; mp_enable_preemption(); return (USER_CTHREAD); } /* * thread_fast_set_cthread_self64: Sets the machine kernel thread ID of the * current thread to the given thread ID; fast version for 64-bit processes * * Parameters: self Thread ID * * Returns: 0 Success * !0 Not success */ kern_return_t thread_fast_set_cthread_self64(uint64_t self) { pcb_t pcb = current_thread()->machine.pcb; cpu_data_t *cdp; /* check for canonical address, set 0 otherwise */ if (!IS_USERADDR64_CANONICAL(self)) self = 0ULL; pcb->cthread_self = self; mp_disable_preemption(); cdp = current_cpu_datap(); #if defined(__x86_64__) if ((cdp->cpu_uber.cu_user_gs_base != pcb->cthread_self) || (pcb->cthread_self != rdmsr64(MSR_IA32_KERNEL_GS_BASE))) wrmsr64(MSR_IA32_KERNEL_GS_BASE, self); #endif cdp->cpu_uber.cu_user_gs_base = self; mp_enable_preemption(); return (USER_CTHREAD); }
/* * Adjust the Universal (Posix) time gradually. */ kern_return_t host_adjust_time( host_t host, time_value_t newadj, time_value_t *oldadj) /* OUT */ { time_value_t oadj; integer_t ndelta; spl_t s; if (host == HOST_NULL) return (KERN_INVALID_HOST); ndelta = (newadj.seconds * 1000000) + newadj.microseconds; #if NCPUS > 1 thread_bind(current_thread(), master_processor); mp_disable_preemption(); if (current_processor() != master_processor) { mp_enable_preemption(); thread_block((void (*)(void)) 0); } else { mp_enable_preemption(); } #endif /* NCPUS > 1 */ s = splclock(); oadj.seconds = timedelta / 1000000; oadj.microseconds = timedelta % 1000000; if (timedelta == 0) { if (ndelta > bigadj) tickdelta = 10 * tickadj; else tickdelta = tickadj; } if (ndelta % tickdelta) ndelta = ndelta / tickdelta * tickdelta; timedelta = ndelta; splx(s); #if NCPUS > 1 thread_bind(current_thread(), PROCESSOR_NULL); #endif /* NCPUS > 1 */ *oldadj = oadj; return (KERN_SUCCESS); }
boolean_t swtch_pri( __unused struct swtch_pri_args *args) { register processor_t myprocessor; boolean_t result; disable_preemption(); myprocessor = current_processor(); if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) { mp_enable_preemption(); return (FALSE); } enable_preemption(); counter(c_swtch_pri_block++); thread_depress_abstime(thread_depress_time); thread_block_reason((thread_continue_t)swtch_pri_continue, NULL, AST_YIELD); thread_depress_abort_internal(current_thread()); disable_preemption(); myprocessor = current_processor(); result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0; enable_preemption(); return (result); }
/* * Acquire a usimple_lock. * * MACH_RT: Returns with preemption disabled. Note * that the hw_lock routines are responsible for * maintaining preemption state. */ void usimple_lock( usimple_lock_t l) { int i; unsigned int timeouttb; /* Used to convert time to timebase ticks */ pc_t pc; #if ETAP_LOCK_TRACE etap_time_t start_wait_time; int no_miss_info = 0; #endif /* ETAP_LOCK_TRACE */ #if USLOCK_DEBUG int count = 0; #endif /* USLOCK_DEBUG */ OBTAIN_PC(pc, l); USLDBG(usld_lock_pre(l, pc)); #if ETAP_LOCK_TRACE ETAP_TIME_CLEAR(start_wait_time); #endif /* ETAP_LOCK_TRACE */ while (!hw_lock_try(&l->interlock)) { ETAPCALL(if (no_miss_info++ == 0) start_wait_time = etap_simplelock_miss(l)); while (hw_lock_held(&l->interlock)) { /* * Spin watching the lock value in cache, * without consuming external bus cycles. * On most SMP architectures, the atomic * instruction(s) used by hw_lock_try * cost much, much more than an ordinary * memory read. */ #if USLOCK_DEBUG if (count++ > max_lock_loops #if MACH_KDB && NCPUS > 1 && l != &kdb_lock #endif /* MACH_KDB && NCPUS > 1 */ ) { if (l == &printf_lock) { return; } mp_disable_preemption(); #if MACH_KDB db_printf("cpu %d looping on simple_lock(%x)" "(=%x)", cpu_number(), l, *hw_lock_addr(l->interlock)); db_printf(" called by %x\n", pc); #endif Debugger("simple lock deadlock detection"); count = 0; mp_enable_preemption(); } #endif /* USLOCK_DEBUG */ } } ETAPCALL(etap_simplelock_hold(l, pc, start_wait_time)); USLDBG(usld_lock_post(l, pc)); }
static void ml_phys_write_data(pmap_paddr_t paddr, unsigned long data, int size) { mapwindow_t *map; mp_disable_preemption(); map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | INTEL_PTE_RW | (paddr & PG_FRAME) | INTEL_PTE_REF | INTEL_PTE_MOD)); switch (size) { case 1: *(unsigned char *)((uintptr_t)map->prv_CADDR | ((uint32_t)paddr & INTEL_OFFMASK)) = (unsigned char)data; break; case 2: *(unsigned short *)((uintptr_t)map->prv_CADDR | ((uint32_t)paddr & INTEL_OFFMASK)) = (unsigned short)data; break; case 4: default: *(unsigned int *)((uintptr_t)map->prv_CADDR | ((uint32_t)paddr & INTEL_OFFMASK)) = data; break; } pmap_put_mapwindow(map); mp_enable_preemption(); }
/* ARGSUSED */ void usld_lock_pre( usimple_lock_t l, pc_t pc) { char caller[] = "usimple_lock"; if (!usld_lock_common_checks(l, caller)) return; /* * Note that we have a weird case where we are getting a lock when we are] * in the process of putting the system to sleep. We are running with no * current threads, therefore we can't tell if we are trying to retake a lock * we have or someone on the other processor has it. Therefore we just * ignore this test if the locking thread is 0. */ if ((l->debug.state & USLOCK_TAKEN) && l->debug.lock_thread && l->debug.lock_thread == (void *) current_thread()) { printf("%s: lock %p already locked (at %p) by", caller, l, l->debug.lock_pc); printf(" current thread %p (new attempt at pc %p)\n", l->debug.lock_thread, pc); panic("%s", caller); } mp_disable_preemption(); usl_trace(l, cpu_number(), pc, caller); mp_enable_preemption(); }
static unsigned int ml_phys_read_data(pmap_paddr_t paddr, int size ) { mapwindow_t *map; unsigned int result; mp_disable_preemption(); map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | (paddr & PG_FRAME) | INTEL_PTE_REF)); switch (size) { unsigned char s1; unsigned short s2; case 1: s1 = *(unsigned char *)((uintptr_t)map->prv_CADDR | ((uint32_t)paddr & INTEL_OFFMASK)); result = s1; break; case 2: s2 = *(unsigned short *)((uintptr_t)map->prv_CADDR | ((uint32_t)paddr & INTEL_OFFMASK)); result = s2; break; case 4: default: result = *(unsigned int *)((uintptr_t)map->prv_CADDR | ((uint32_t)paddr & INTEL_OFFMASK)); break; } pmap_put_mapwindow(map); mp_enable_preemption(); return result; }
void bcopy_phys( addr64_t src64, addr64_t dst64, vm_size_t bytes) { mapwindow_t *src_map, *dst_map; /* ensure we stay within a page */ if ( ((((uint32_t)src64 & (NBPG-1)) + bytes) > NBPG) || ((((uint32_t)dst64 & (NBPG-1)) + bytes) > NBPG) ) { panic("bcopy_phys alignment"); } mp_disable_preemption(); src_map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | ((pmap_paddr_t)src64 & PG_FRAME) | INTEL_PTE_REF)); dst_map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | INTEL_PTE_RW | ((pmap_paddr_t)dst64 & PG_FRAME) | INTEL_PTE_REF | INTEL_PTE_MOD)); bcopy((void *) ((uintptr_t)src_map->prv_CADDR | ((uint32_t)src64 & INTEL_OFFMASK)), (void *) ((uintptr_t)dst_map->prv_CADDR | ((uint32_t)dst64 & INTEL_OFFMASK)), bytes); pmap_put_mapwindow(src_map); pmap_put_mapwindow(dst_map); mp_enable_preemption(); }
/* * start_timer starts the given timer for this cpu. It is called * exactly once for each cpu during the boot sequence. */ void start_timer( register timer_t timer) { timer->tstamp = get_timestamp(); mp_disable_preemption(); current_timer[cpu_number()] = timer; mp_enable_preemption(); }
/* * Debug checks on a usimple_lock just before * attempting to acquire it. * * Preemption isn't guaranteed to be disabled. */ void usld_lock_try_pre( usimple_lock_t l, pc_t pc) { char caller[] = "usimple_lock_try"; if (!usld_lock_common_checks(l, caller)) return; mp_disable_preemption(); usl_trace(l, cpu_number(), pc, caller); mp_enable_preemption(); }
/* * Set the Universal (Posix) time. Privileged call. */ kern_return_t host_set_time( host_t host, time_value_t new_time) { spl_t s; if (host == HOST_NULL) return(KERN_INVALID_HOST); #if NCPUS > 1 thread_bind(current_thread(), master_processor); mp_disable_preemption(); if (current_processor() != master_processor) { mp_enable_preemption(); thread_block((void (*)(void)) 0); } else { mp_enable_preemption(); } #endif /* NCPUS > 1 */ s = splhigh(); time = new_time; update_mapped_time(&time); #if PTIME_MACH_RT rtc_gettime_interrupts_disabled((tvalspec_t *)&last_utime_tick); #endif /* PTIME_MACH_RT */ #if 0 (void)bbc_settime((time_value_t *)&time); #endif splx(s); #if NCPUS > 1 thread_bind(current_thread(), PROCESSOR_NULL); #endif /* NCPUS > 1 */ return (KERN_SUCCESS); }
static void swtch_pri_continue(void) { register processor_t myprocessor; boolean_t result; thread_depress_abort_internal(current_thread()); disable_preemption(); myprocessor = current_processor(); result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0; mp_enable_preemption(); thread_syscall_return(result); /*NOTREACHED*/ }
static void ml_phys_write_long_long(pmap_paddr_t paddr, unsigned long long data) { mapwindow_t *map; mp_disable_preemption(); map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | INTEL_PTE_RW | (paddr & PG_FRAME) | INTEL_PTE_REF | INTEL_PTE_MOD)); *(unsigned long long *)((uintptr_t)map->prv_CADDR | ((uint32_t)paddr & INTEL_OFFMASK)) = data; pmap_put_mapwindow(map); mp_enable_preemption(); }
void bzero_phys( addr64_t src64, uint32_t bytes) { mapwindow_t *map; mp_disable_preemption(); map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | INTEL_PTE_RW | ((pmap_paddr_t)src64 & PG_FRAME) | INTEL_PTE_REF | INTEL_PTE_MOD)); bzero((void *)((uintptr_t)map->prv_CADDR | ((uint32_t)src64 & INTEL_OFFMASK)), bytes); pmap_put_mapwindow(map); mp_enable_preemption(); }
/* * time_int_exit does interrupt exit timing. Caller must lock out * interrupts and take a timestamp. ts is a timestamp taken after * interrupts were locked out. old_timer is the timer value pushed * onto the stack or otherwise saved after time_int_entry returned * it. */ void time_int_exit( unsigned ts, timer_t old_timer) { int elapsed; int mycpu; timer_t mytimer; mp_disable_preemption(); /* * Calculate elapsed time. */ mycpu = cpu_number(); mytimer = current_timer[mycpu]; elapsed = ts - mytimer->tstamp; #ifdef TIMER_MAX if (elapsed < 0) elapsed += TIMER_MAX; #endif /* TIMER_MAX */ /* * Update current timer. */ mytimer->low_bits += elapsed; mytimer->tstamp = 0; /* * If normalization requested, do it. */ if (mytimer->low_bits & TIMER_LOW_FULL) { timer_normalize(mytimer); } if (old_timer->low_bits & TIMER_LOW_FULL) { timer_normalize(old_timer); } /* * Start timer that was running before interrupt. */ old_timer->tstamp = ts; current_timer[mycpu] = old_timer; mp_enable_preemption(); }
static unsigned long long ml_phys_read_long_long(pmap_paddr_t paddr ) { mapwindow_t *map; unsigned long long result; mp_disable_preemption(); map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | (paddr & PG_FRAME) | INTEL_PTE_REF)); result = *(unsigned long long *)((uintptr_t)map->prv_CADDR | ((uint32_t)paddr & INTEL_OFFMASK)); pmap_put_mapwindow(map); mp_enable_preemption(); return result; }
/* * Determine whether any usimple_locks are currently held. * * MACH_RT: Caller's preemption state is uncertain. If * preemption has been disabled, this check is accurate. * Otherwise, this check is just a guess. We do the best * we can by disabling scheduler interrupts, so at least * the check is accurate w.r.t. whatever cpu we're running * on while in this routine. */ void usld_lock_none_held() { register int mycpu; spl_t s; unsigned int locks_held; char *caller = "usimple_lock_none_held"; DISABLE_INTERRUPTS(s); mp_disable_preemption(); mycpu = cpu_number(); locks_held = uslock_stack_index[mycpu]; mp_enable_preemption(); ENABLE_INTERRUPTS(s); if (locks_held > 0) panic("%s: no locks should be held (0x%x locks held)", caller, (integer_t) locks_held); }
/* * timer_switch switches to a new timer. The machine * dependent routine/macro get_timestamp must return a timestamp. * Caller must lock out interrupts. */ void timer_switch( timer_t new_timer) { int elapsed; int mycpu; timer_t mytimer; unsigned ts; mp_disable_preemption(); /* * Calculate elapsed time. */ mycpu = cpu_number(); mytimer = current_timer[mycpu]; ts = get_timestamp(); elapsed = ts - mytimer->tstamp; #ifdef TIMER_MAX if (elapsed < 0) elapsed += TIMER_MAX; #endif /* TIMER_MAX */ /* * Update current timer. */ mytimer->low_bits += elapsed; mytimer->tstamp = 0; /* * Normalization check */ if (mytimer->low_bits & TIMER_LOW_FULL) { timer_normalize(mytimer); } /* * Record new timer. */ current_timer[mycpu] = new_timer; new_timer->tstamp = ts; mp_enable_preemption(); }
/* * init_timers initializes all non-thread timers and puts the * service routine on the callout queue. All timers must be * serviced by the callout routine once an hour. */ void init_timers(void) { register int i; register timer_t this_timer; /* * Initialize all the kernel timers and start the one * for this cpu (master) slaves start theirs later. */ this_timer = &kernel_timer[0]; for ( i=0 ; i<NCPUS ; i++, this_timer++) { timer_init(this_timer); current_timer[i] = (timer_t) 0; } mp_disable_preemption(); start_timer(&kernel_timer[cpu_number()]); mp_enable_preemption(); }
void fillPage(ppnum_t pa, unsigned int fill) { mapwindow_t *map; pmap_paddr_t src; int i; int cnt = PAGE_SIZE/sizeof(unsigned int); unsigned int *addr; mp_disable_preemption(); src = i386_ptob(pa); map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | INTEL_PTE_RW | (src & PG_FRAME) | INTEL_PTE_REF | INTEL_PTE_MOD)); for (i = 0, addr = (unsigned int *)map->prv_CADDR; i < cnt ; i++ ) *addr++ = fill; pmap_put_mapwindow(map); mp_enable_preemption(); }
void delayed_clock(void) { int i; int my_cpu; mp_disable_preemption(); my_cpu = cpu_number(); if (missed_clock[my_cpu] > 1 && detect_lost_tick) printf("hardclock: missed %d clock interrupt(s) at %x\n", missed_clock[my_cpu]-1, masked_pc[my_cpu]); if (my_cpu == master_cpu) { i = rtclock_intr(); assert(i == 0); } hertz_tick(0, masked_pc[my_cpu]); missed_clock[my_cpu] = 0; mp_enable_preemption(); }
/* * time_trap_uexit does trap exit timing. Caller must lock out * interrupts and take a timestamp. ts is a timestamp taken after * interrupts were locked out. Must only be called if returning to * user mode. */ void time_trap_uexit( unsigned ts) { int elapsed; int mycpu; timer_t mytimer; mp_disable_preemption(); /* * Calculate elapsed time. */ mycpu = cpu_number(); mytimer = current_timer[mycpu]; elapsed = ts - mytimer->tstamp; #ifdef TIMER_MAX if (elapsed < 0) elapsed += TIMER_MAX; #endif /* TIMER_MAX */ /* * Update current timer. */ mytimer->low_bits += elapsed; mytimer->tstamp = 0; if (mytimer->low_bits & TIMER_LOW_FULL) { timer_normalize(mytimer); /* SYSTEMMODE */ } mytimer = &(current_thread()->user_timer); /* * Record new timer. */ current_timer[mycpu] = mytimer; mytimer->tstamp = ts; mp_enable_preemption(); }
void lapic_shutdown(void) { uint32_t lo; uint32_t hi; uint32_t value; /* Shutdown if local APIC was enabled by OS */ if (lapic_os_enabled == FALSE) return; mp_disable_preemption(); /* ExtINT: masked */ if (get_cpu_number() == master_cpu) { value = LAPIC_READ(LVT_LINT0); value |= LAPIC_LVT_MASKED; LAPIC_WRITE(LVT_LINT0, value); } /* Error: masked */ LAPIC_WRITE(LVT_ERROR, LAPIC_READ(LVT_ERROR) | LAPIC_LVT_MASKED); /* Timer: masked */ LAPIC_WRITE(LVT_TIMER, LAPIC_READ(LVT_TIMER) | LAPIC_LVT_MASKED); /* Perfmon: masked */ LAPIC_WRITE(LVT_PERFCNT, LAPIC_READ(LVT_PERFCNT) | LAPIC_LVT_MASKED); /* APIC software disabled */ LAPIC_WRITE(SVR, LAPIC_READ(SVR) & ~LAPIC_SVR_ENABLE); /* Bypass the APIC completely and update cpu features */ rdmsr(MSR_IA32_APIC_BASE, lo, hi); lo &= ~MSR_IA32_APIC_BASE_ENABLE; wrmsr(MSR_IA32_APIC_BASE, lo, hi); cpuid_set_info(); mp_enable_preemption(); }
void xpr( const char *msg, long arg1, long arg2, long arg3, long arg4, long arg5) { spl_t s; struct xprbuf *x; /* If we aren't initialized, ignore trace request */ if (!xprenable || (xprptr == 0)) return; /* Guard against all interrupts and allocate next buffer. */ s = splhigh(); simple_lock(&xprlock); x = xprptr++; if (xprptr >= xprlast) { /* wrap around */ xprptr = xprbase; } /* Save xprptr in allocated memory. */ *(struct xprbuf **)xprlast = xprptr; simple_unlock(&xprlock); x->timestamp = XPR_TIMESTAMP; splx(s); x->msg = msg; x->arg1 = arg1; x->arg2 = arg2; x->arg3 = arg3; x->arg4 = arg4; x->arg5 = arg5; mp_disable_preemption(); x->cpuinfo = cpu_number(); mp_enable_preemption(); }
/* * thread_set_user_ldt routine is the interface for the user level * settable ldt entry feature. allowing a user to create arbitrary * ldt entries seems to be too large of a security hole, so instead * this mechanism is in place to allow user level processes to have * an ldt entry that can be used in conjunction with the FS register. * * Swapping occurs inside the pcb.c file along with initialization * when a thread is created. The basic functioning theory is that the * pcb->uldt_selector variable will contain either 0 meaning the * process has not set up any entry, or the selector to be used in * the FS register. pcb->uldt_desc contains the actual descriptor the * user has set up stored in machine usable ldt format. * * Currently one entry is shared by all threads (USER_SETTABLE), but * this could be changed in the future by changing how this routine * allocates the selector. There seems to be no real reason at this * time to have this added feature, but in the future it might be * needed. * * address is the linear address of the start of the data area size * is the size in bytes of the area flags should always be set to 0 * for now. in the future it could be used to set R/W permisions or * other functions. Currently the segment is created as a data segment * up to 1 megabyte in size with full read/write permisions only. * * this call returns the segment selector or -1 if any error occurs */ kern_return_t thread_set_user_ldt(uint32_t address, uint32_t size, uint32_t flags) { pcb_t pcb; struct fake_descriptor temp; int mycpu; if (flags != 0) return -1; // flags not supported if (size > 0xFFFFF) return -1; // size too big, 1 meg is the limit mp_disable_preemption(); mycpu = cpu_number(); // create a "fake" descriptor so we can use fix_desc() // to build a real one... // 32 bit default operation size // standard read/write perms for a data segment pcb = (pcb_t)current_thread()->machine.pcb; temp.offset = address; temp.lim_or_seg = size; temp.size_or_wdct = SZ_32; temp.access = ACC_P|ACC_PL_U|ACC_DATA_W; // turn this into a real descriptor fix_desc(&temp,1); // set up our data in the pcb pcb->uldt_desc = *(struct real_descriptor*)&temp; pcb->uldt_selector = USER_SETTABLE; // set the selector value // now set it up in the current table... *ldt_desc_p(USER_SETTABLE) = *(struct real_descriptor*)&temp; mp_enable_preemption(); return USER_SETTABLE; }
/* ARGSUSED */ void usld_lock_pre( usimple_lock_t l, pc_t pc) { char *caller = "usimple_lock"; if (!usld_lock_common_checks(l, caller)) return; if ((l->debug.state & USLOCK_TAKEN) && l->debug.lock_thread == (void *) current_thread()) { printf("%s: lock 0x%x already locked (at 0x%x) by", caller, (integer_t) l, l->debug.lock_pc); printf(" current thread 0x%x (new attempt at pc 0x%x)\n", l->debug.lock_thread, pc); panic(caller); } mp_disable_preemption(); usl_trace(l, cpu_number(), pc, caller); mp_enable_preemption(); }
/* * time_int_entry does interrupt entry timing. Caller must lock out * interrupts and take a timestamp. ts is a timestamp taken after * interrupts were locked out. new_timer is the new timer to * switch to. This routine returns the currently running timer, * which MUST be pushed onto the stack by the caller, or otherwise * saved for time_int_exit. */ timer_t time_int_entry( unsigned ts, timer_t new_timer) { int elapsed; int mycpu; timer_t mytimer; mp_disable_preemption(); /* * Calculate elapsed time. */ mycpu = cpu_number(); mytimer = current_timer[mycpu]; elapsed = ts - mytimer->tstamp; #ifdef TIMER_MAX if (elapsed < 0) elapsed += TIMER_MAX; #endif /* TIMER_MAX */ /* * Update current timer. */ mytimer->low_bits += elapsed; mytimer->tstamp = 0; /* * Switch to new timer, and save old one on stack. */ new_timer->tstamp = ts; current_timer[mycpu] = new_timer; mp_enable_preemption(); return(mytimer); }
void ast_check(void) { register int mycpu; register processor_t myprocessor; register thread_t thread = current_thread(); spl_t s = splsched(); mp_disable_preemption(); mycpu = cpu_number(); /* * Check processor state for ast conditions. */ myprocessor = cpu_to_processor(mycpu); switch(myprocessor->state) { case PROCESSOR_OFF_LINE: case PROCESSOR_IDLE: case PROCESSOR_DISPATCHING: /* * No ast. */ break; #if NCPUS > 1 case PROCESSOR_ASSIGN: case PROCESSOR_SHUTDOWN: /* * Need ast to force action thread onto processor. * * XXX Should check if action thread is already there. */ ast_on(mycpu, AST_BLOCK); break; #endif /* NCPUS > 1 */ case PROCESSOR_RUNNING: case PROCESSOR_VIDLE: /* * Propagate thread ast to processor. If we already * need an ast, don't look for more reasons. */ ast_propagate(current_act(), mycpu); if (ast_needed(mycpu)) break; /* * Context switch check. */ if (csw_needed(thread, myprocessor)) { ast_on(mycpu, (myprocessor->first_quantum ? AST_BLOCK : AST_QUANTUM)); } break; default: panic("ast_check: Bad processor state"); } mp_enable_preemption(); splx(s); }
__private_extern__ int ml_copy_phys(addr64_t src64, addr64_t dst64, vm_size_t bytes) { void *src, *dst; int err = 0; mp_disable_preemption(); #if NCOPY_WINDOWS > 0 mapwindow_t *src_map, *dst_map; /* We rely on MTRRs here */ src_map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | ((pmap_paddr_t)src64 & PG_FRAME) | INTEL_PTE_REF)); dst_map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | INTEL_PTE_RW | ((pmap_paddr_t)dst64 & PG_FRAME) | INTEL_PTE_REF | INTEL_PTE_MOD)); src = (void *) ((uintptr_t)src_map->prv_CADDR | ((uint32_t)src64 & INTEL_OFFMASK)); dst = (void *) ((uintptr_t)dst_map->prv_CADDR | ((uint32_t)dst64 & INTEL_OFFMASK)); #elif defined(__x86_64__) addr64_t debug_pa = 0; /* If either destination or source are outside the * physical map, establish a physical window onto the target frame. */ assert(physmap_enclosed(src64) || physmap_enclosed(dst64)); if (physmap_enclosed(src64) == FALSE) { src = (void *)(debugger_window_kva | (src64 & INTEL_OFFMASK)); dst = PHYSMAP_PTOV(dst64); debug_pa = src64 & PG_FRAME; } else if (physmap_enclosed(dst64) == FALSE) { src = PHYSMAP_PTOV(src64); dst = (void *)(debugger_window_kva | (dst64 & INTEL_OFFMASK)); debug_pa = dst64 & PG_FRAME; } else { src = PHYSMAP_PTOV(src64); dst = PHYSMAP_PTOV(dst64); } /* DRK: debugger only routine, we don't bother checking for an * identical mapping. */ if (debug_pa) { if (debugger_window_kva == 0) panic("%s: invoked in non-debug mode", __FUNCTION__); /* Establish a cache-inhibited physical window; some platforms * may not cover arbitrary ranges with MTRRs */ pmap_store_pte(debugger_ptep, debug_pa | INTEL_PTE_NCACHE | INTEL_PTE_RW | INTEL_PTE_REF| INTEL_PTE_MOD | INTEL_PTE_VALID); flush_tlb_raw(); #if DEBUG kprintf("Remapping debugger physical window at %p to 0x%llx\n", (void *)debugger_window_kva, debug_pa); #endif } #endif /* ensure we stay within a page */ if (((((uint32_t)src64 & (I386_PGBYTES-1)) + bytes) > I386_PGBYTES) || ((((uint32_t)dst64 & (I386_PGBYTES-1)) + bytes) > I386_PGBYTES) ) { panic("ml_copy_phys spans pages, src: 0x%llx, dst: 0x%llx", src64, dst64); } /* * For device register access from the debugger, * 2-byte/16-bit, 4-byte/32-bit and 8-byte/64-bit copies are handled * by assembly routines ensuring the required access widths. * 1-byte and other copies are handled by the regular _bcopy. */ switch (bytes) { case 2: err = _bcopy2(src, dst); break; case 4: err = _bcopy4(src, dst); break; case 8: err = _bcopy8(src, dst); break; case 1: default: err = _bcopy(src, dst, bytes); break; } #if NCOPY_WINDOWS > 0 pmap_put_mapwindow(src_map); pmap_put_mapwindow(dst_map); #endif mp_enable_preemption(); return err; }
void ast_taken( boolean_t preemption, ast_t mask, spl_t old_spl #if FAST_IDLE ,int thread_type #endif /* FAST_IDLE */ ) { register thread_t self = current_thread(); register processor_t mypr; register ast_t reasons; register int mycpu; thread_act_t act = self->top_act; /* * Interrupts are still disabled. * We must clear need_ast and then enable interrupts. */ extern void log_thread_action(thread_t, char *); #if 0 log_thread_action (current_thread(), "ast_taken"); #endif mp_disable_preemption(); mycpu = cpu_number(); reasons = need_ast[mycpu] & mask; need_ast[mycpu] &= ~reasons; mp_enable_preemption(); splx(old_spl); /* * These actions must not block. */ #if MCMSG if (reasons & AST_MCMSG) mcmsg_ast(); #endif /* MCMSG */ if (reasons & AST_NETWORK) net_ast(); #if MCMSG_ENG if (reasons & AST_RPCREQ) rpc_engine_request_intr(); if (reasons & AST_RPCREPLY) rpc_engine_reply_intr(); if (reasons & AST_RPCDEPART) rpc_engine_depart_intr(); if (reasons & AST_RDMASEND) rdma_engine_send_intr(); if (reasons & AST_RDMARECV) rdma_engine_recv_intr(); if (reasons & AST_RDMATXF) rdma_engine_send_fault_intr(); if (reasons & AST_RDMARXF) rdma_engine_recv_fault_intr(); #endif /* MCMSG_ENG */ #if PARAGON860 && MCMSG_ENG if (reasons & AST_SCAN_INPUT) scan_input_ast(); #endif /* PARAGON860 */ #if DIPC if (reasons & AST_DIPC) dipc_ast(); #endif /* DIPC */ /* * Make darn sure that we don't call thread_halt_self * or thread_block from the idle thread. */ /* XXX - this isn't currently right for the HALT case... */ mp_disable_preemption(); mypr = current_processor(); if (self == mypr->idle_thread) { #if NCPUS == 1 if (reasons & AST_URGENT) { if (!preemption) panic("ast_taken: AST_URGENT for idle_thr w/o preemption"); } #endif mp_enable_preemption(); return; } mp_enable_preemption(); #if FAST_IDLE if (thread_type != NO_IDLE_THREAD) return; #endif /* FAST_IDLE */ #if TASK_SWAPPER /* must be before AST_APC */ if (reasons & AST_SWAPOUT) { spl_t s; swapout_ast(); s = splsched(); mp_disable_preemption(); mycpu = cpu_number(); if (need_ast[mycpu] & AST_APC) { /* generated in swapout_ast() to get suspended */ reasons |= AST_APC; /* process now ... */ need_ast[mycpu] &= ~AST_APC; /* ... and not later */ } mp_enable_preemption(); splx(s); } #endif /* TASK_SWAPPER */ /* migration APC hook */ if (reasons & AST_APC) { act_execute_returnhandlers(); return; /* auto-retry will catch anything new */ } /* * thread_block needs to know if the thread's quantum * expired so the thread can be put on the tail of * run queue. One of the previous actions might well * have woken a high-priority thread, so we also use * csw_needed check. */ { void (*safept)(void) = (void (*)(void))SAFE_EXCEPTION_RETURN; if (reasons &= AST_PREEMPT) { if (preemption) safept = (void (*)(void)) 0; } else { mp_disable_preemption(); mypr = current_processor(); if (csw_needed(self, mypr)) { reasons = (mypr->first_quantum ? AST_BLOCK : AST_QUANTUM); } mp_enable_preemption(); } if (reasons) { counter(c_ast_taken_block++); thread_block_reason(safept, reasons); } } }