/**
* __hwspin_trylock() - attempt to lock a specific hwspinlock
* @hwlock: an hwspinlock which we want to trylock
* @mode: controls whether local interrupts are disabled or not
* @flags: a pointer where the caller's interrupt state will be saved at (if
* requested)
*
* This function attempts to lock an hwspinlock, and will immediately
* fail if the hwspinlock is already taken.
*
* Upon a successful return from this function, preemption (and possibly
* interrupts) is disabled, so the caller must not sleep, and is advised to
* release the hwspinlock as soon as possible. This is required in order to
* minimize remote cores polling on the hardware interconnect.
*
* The user decides whether local interrupts are disabled or not, and if yes,
* whether he wants their previous state to be saved. It is up to the user
* to choose the appropriate @mode of operation, exactly the same way users
* should decide between spin_trylock, spin_trylock_irq and
* spin_trylock_irqsave.
*
* Returns 0 if we successfully locked the hwspinlock or -EBUSY if
* the hwspinlock was already taken.
* This function will never sleep.
*/
int __hwspin_trylock(struct hwspinlock *hwlock, int mode, unsigned long *flags)
{
int ret;
BUG_ON(!hwlock);
BUG_ON(!flags && mode == HWLOCK_IRQSTATE);
/*
* This spin_lock{_irq, _irqsave} serves three purposes:
*
* 1. Disable preemption, in order to minimize the period of time
* in which the hwspinlock is taken. This is important in order
* to minimize the possible polling on the hardware interconnect
* by a remote user of this lock.
* 2. Make the hwspinlock SMP-safe (so we can take it from
* additional contexts on the local host).
* 3. Ensure that in_atomic/might_sleep checks catch potential
* problems with hwspinlock usage (e.g. scheduler checks like
* 'scheduling while atomic' etc.)
*/
if (mode == HWLOCK_IRQSTATE)
ret = spin_trylock_irqsave(&hwlock->lock, *flags);
else if (mode == HWLOCK_IRQ)
ret = spin_trylock_irq(&hwlock->lock);
else
ret = spin_trylock(&hwlock->lock);
/* is lock already taken by another context on the local cpu ? */
if (!ret)
return -EBUSY;
/* try to take the hwspinlock device */
ret = hwlock->bank->ops->trylock(hwlock);
/* if hwlock is already taken, undo spin_trylock_* and exit */
if (!ret) {
if (mode == HWLOCK_IRQSTATE)
spin_unlock_irqrestore(&hwlock->lock, *flags);
else if (mode == HWLOCK_IRQ)
spin_unlock_irq(&hwlock->lock);
else
spin_unlock(&hwlock->lock);
return -EBUSY;
}
/*
* We can be sure the other core's memory operations
* are observable to us only _after_ we successfully take
* the hwspinlock, and we must make sure that subsequent memory
* operations (both reads and writes) will not be reordered before
* we actually took the hwspinlock.
*
* Note: the implicit memory barrier of the spinlock above is too
* early, so we need this additional explicit memory barrier.
*/
mb();
return 0;
}
/**
* x86_64 specific code for carrying out inter-CPU function calls.
* This function should not be called directly. Call xcall_function() instead.
*
* Arguments:
* [IN] cpu_mask: The target CPUs of the cross-call.
* [IN] func: The function to execute on each target CPU.
* [IN] info: Argument to pass to func().
* [IN] wait: true = wait for cross-call to fully complete.
*
* Returns:
* Success: 0
* Failure: Error code
*/
int
arch_xcall_function(
cpumask_t cpu_mask,
void (*func)(void *info),
void * info,
bool wait
)
{
struct xcall_data_struct data;
unsigned int num_cpus;
unsigned int cpu;
BUG_ON(irqs_disabled());
/* Count how many CPUs are being targeted */
num_cpus = cpus_weight(cpu_mask);
if (!num_cpus)
return 0;
/* Fill in the xcall data structure on our stack */
data.func = func;
data.info = info;
atomic_set(&data.started, 0);
if (wait)
atomic_set(&data.finished, 0);
data.wait = wait;
/* Spin with IRQs enabled */
while (!spin_trylock_irq(&xcall_data_lock))
;
/* IRQs are now disabled */
/* Set the global xcall data pointer */
xcall_data = &data;
wmb();
/* Send inter-processor interrupts to the target CPUs */
for_each_cpu_mask(cpu, cpu_mask)
lapic_send_ipi(cpu, XCALL_FUNCTION_VECTOR);
/* Wait for initiation responses */
while (atomic_read(&data.started) != num_cpus)
cpu_relax();
/* If requested, wait for completion responses */
if (wait) {
while (atomic_read(&data.finished) != num_cpus)
cpu_relax();
}
spin_unlock_irq(&xcall_data_lock);
return 0;
}
void mlx4_en_poll_tx_cq(unsigned long data)
{
struct mlx4_en_cq *cq = (struct mlx4_en_cq *) data;
struct mlx4_en_priv *priv = netdev_priv(cq->dev);
struct mlx4_en_tx_ring *ring = &priv->tx_ring[cq->ring];
u32 inflight;
INC_PERF_COUNTER(priv->pstats.tx_poll);
if (!spin_trylock_irq(&ring->comp_lock)) {
mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT);
return;
}
mlx4_en_process_tx_cq(cq->dev, cq);
inflight = (u32) (ring->prod - ring->cons - ring->last_nr_txbb);
/* If there are still packets in flight and the timer has not already
* been scheduled by the Tx routine then schedule it here to guarantee
* completion processing of these packets */
if (inflight && priv->port_up)
mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT);
spin_unlock_irq(&ring->comp_lock);
}
void mlx4_en_poll_tx_cq(unsigned long data)
{
struct mlx4_en_cq *cq = (struct mlx4_en_cq *) data;
struct mlx4_en_priv *priv = netdev_priv(cq->dev);
struct mlx4_en_tx_ring *ring = &priv->tx_ring[cq->ring];
u32 inflight;
INC_PERF_COUNTER(priv->pstats.tx_poll);
if (!spin_trylock_irq(&ring->comp_lock)) {
mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT);
return;
}
mlx4_en_process_tx_cq(cq->dev, cq);
inflight = (u32) (ring->prod - ring->cons - ring->last_nr_txbb);
/*
*/
if (inflight && priv->port_up)
mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT);
spin_unlock_irq(&ring->comp_lock);
}
