int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
int retry, int wait)
{
int ret, me;
/*
* Can die spectacularly if this CPU isn't yet marked online
*/
if (!cpu_online(cpu))
return 0;
me = get_cpu();
BUG_ON(!cpu_online(me));
if (cpu == me) {
local_irq_disable();
func(info);
local_irq_enable();
put_cpu();
return 0;
}
ret = smp_call_function_mask(cpumask_of_cpu(cpu), func, info, retry,
wait);
put_cpu();
return 0;
}
void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
{
cpumask_t cpu_mask;
unsigned int pid;
preempt_disable();
pid = vma ? vma->vm_mm->context.id : 0;
if (unlikely(pid == MMU_NO_CONTEXT))
goto bail;
cpu_mask = vma->vm_mm->cpu_vm_mask;
cpu_clear(smp_processor_id(), cpu_mask);
if (!cpus_empty(cpu_mask)) {
/* If broadcast tlbivax is supported, use it */
if (mmu_has_feature(MMU_FTR_USE_TLBIVAX_BCAST)) {
int lock = mmu_has_feature(MMU_FTR_LOCK_BCAST_INVAL);
if (lock)
spin_lock(&tlbivax_lock);
_tlbivax_bcast(vmaddr, pid);
if (lock)
spin_unlock(&tlbivax_lock);
goto bail;
} else {
struct tlb_flush_param p = { .pid = pid, .addr = vmaddr };
smp_call_function_mask(cpu_mask,
do_flush_tlb_page_ipi, &p, 1);
}
}
_tlbil_va(vmaddr, pid);
bail:
preempt_enable();
}
/**
* smp_call_function_single - Run a function on a specific CPU
* @cpu: The target CPU. Cannot be the calling CPU.
* @func: The function to run. This must be fast and non-blocking.
* @info: An arbitrary pointer to pass to the function.
* @nonatomic: Unused.
* @wait: If true, wait until function has completed on other CPUs.
*
* Returns 0 on success, else a negative status code.
*
* If @wait is true, then returns once @func has returned; otherwise
* it returns just before the target cpu calls @func.
*/
int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
int nonatomic, int wait)
{
/* prevent preemption and reschedule on another processor */
int ret;
int me = get_cpu();
if (cpu == me) {
local_irq_disable();
func(info);
local_irq_enable();
put_cpu();
return 0;
}
ret = smp_call_function_mask(cpumask_of_cpu(cpu), func, info, wait);
put_cpu();
return ret;
}
void flush_tlb_mm(struct mm_struct *mm)
{
cpumask_t cpu_mask;
unsigned int pid;
preempt_disable();
pid = mm->context.id;
if (unlikely(pid == MMU_NO_CONTEXT))
goto no_context;
cpu_mask = mm->cpu_vm_mask;
cpu_clear(smp_processor_id(), cpu_mask);
if (!cpus_empty(cpu_mask)) {
struct tlb_flush_param p = { .pid = pid };
smp_call_function_mask(cpu_mask, do_flush_tlb_mm_ipi, &p, 1);
}
_tlbil_pid(pid);
no_context:
preempt_enable();
}
/**
* smp_call_function(): Run a function on all other CPUs.
* @func: The function to run. This must be fast and non-blocking.
* @info: An arbitrary pointer to pass to the function.
* @nonatomic: Unused.
* @wait: If true, wait (atomically) until function has completed on other CPUs.
*
* Returns 0 on success, else a negative status code.
*
* If @wait is true, then returns once @func has returned; otherwise
* it returns just before the target cpu calls @func.
*
* You must not call this function with disabled interrupts or from a
* hardware interrupt handler or from a bottom half handler.
*/
int smp_call_function(void (*func) (void *info), void *info, int nonatomic,
int wait)
{
return smp_call_function_mask(cpu_online_map, func, info, wait);
}
RTDECL(int) RTMpOnPair(RTCPUID idCpu1, RTCPUID idCpu2, uint32_t fFlags, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
{
IPRT_LINUX_SAVE_EFL_AC();
int rc;
RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
AssertReturn(idCpu1 != idCpu2, VERR_INVALID_PARAMETER);
AssertReturn(!(fFlags & RTMPON_F_VALID_MASK), VERR_INVALID_FLAGS);
/*
* Check that both CPUs are online before doing the broadcast call.
*/
RTThreadPreemptDisable(&PreemptState);
if ( RTMpIsCpuOnline(idCpu1)
&& RTMpIsCpuOnline(idCpu2))
{
/*
* Use the smp_call_function variant taking a cpu mask where available,
* falling back on broadcast with filter. Slight snag if one of the
* CPUs is the one we're running on, we must do the call and the post
* call wait ourselves.
*/
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
cpumask_t DstCpuMask;
#endif
RTCPUID idCpuSelf = RTMpCpuId();
bool const fCallSelf = idCpuSelf == idCpu1 || idCpuSelf == idCpu2;
RTMPARGS Args;
Args.pfnWorker = pfnWorker;
Args.pvUser1 = pvUser1;
Args.pvUser2 = pvUser2;
Args.idCpu = idCpu1;
Args.idCpu2 = idCpu2;
Args.cHits = 0;
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28)
cpumask_clear(&DstCpuMask);
cpumask_set_cpu(idCpu1, &DstCpuMask);
cpumask_set_cpu(idCpu2, &DstCpuMask);
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
cpus_clear(DstCpuMask);
cpu_set(idCpu1, DstCpuMask);
cpu_set(idCpu2, DstCpuMask);
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)
smp_call_function_many(&DstCpuMask, rtmpLinuxWrapperPostInc, &Args, !fCallSelf /* wait */);
rc = 0;
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28)
rc = smp_call_function_many(&DstCpuMask, rtmpLinuxWrapperPostInc, &Args, !fCallSelf /* wait */);
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
rc = smp_call_function_mask(DstCpuMask, rtmpLinuxWrapperPostInc, &Args, !fCallSelf /* wait */);
#else /* older kernels */
rc = smp_call_function(rtMpLinuxOnPairWrapper, &Args, 0 /* retry */, !fCallSelf /* wait */);
#endif /* older kernels */
Assert(rc == 0);
/* Call ourselves if necessary and wait for the other party to be done. */
if (fCallSelf)
{
uint32_t cLoops = 0;
rtmpLinuxWrapper(&Args);
while (ASMAtomicReadU32(&Args.cHits) < 2)
{
if ((cLoops & 0x1ff) == 0 && !RTMpIsCpuOnline(idCpuSelf == idCpu1 ? idCpu2 : idCpu1))
break;
cLoops++;
ASMNopPause();
}
}
Assert(Args.cHits <= 2);
if (Args.cHits == 2)
rc = VINF_SUCCESS;
else if (Args.cHits == 1)
rc = VERR_NOT_ALL_CPUS_SHOWED;
else if (Args.cHits == 0)
rc = VERR_CPU_OFFLINE;
else
rc = VERR_CPU_IPE_1;
}
/*
* A CPU must be present to be considered just offline.
*/
else if ( RTMpIsCpuPresent(idCpu1)
&& RTMpIsCpuPresent(idCpu2))
rc = VERR_CPU_OFFLINE;
else
rc = VERR_CPU_NOT_FOUND;
RTThreadPreemptRestore(&PreemptState);;
IPRT_LINUX_RESTORE_EFL_AC();
return rc;
}
