/** * __hwspin_trylock() - attempt to lock a specific hwspinlock * @hwlock: an hwspinlock which we want to trylock * @mode: controls whether local interrupts are disabled or not * @flags: a pointer where the caller's interrupt state will be saved at (if * requested) * * This function attempts to lock an hwspinlock, and will immediately * fail if the hwspinlock is already taken. * * Upon a successful return from this function, preemption (and possibly * interrupts) is disabled, so the caller must not sleep, and is advised to * release the hwspinlock as soon as possible. This is required in order to * minimize remote cores polling on the hardware interconnect. * * The user decides whether local interrupts are disabled or not, and if yes, * whether he wants their previous state to be saved. It is up to the user * to choose the appropriate @mode of operation, exactly the same way users * should decide between spin_trylock, spin_trylock_irq and * spin_trylock_irqsave. * * Returns 0 if we successfully locked the hwspinlock or -EBUSY if * the hwspinlock was already taken. * This function will never sleep. */ int __hwspin_trylock(struct hwspinlock *hwlock, int mode, unsigned long *flags) { int ret; BUG_ON(!hwlock); BUG_ON(!flags && mode == HWLOCK_IRQSTATE); /* * This spin_lock{_irq, _irqsave} serves three purposes: * * 1. Disable preemption, in order to minimize the period of time * in which the hwspinlock is taken. This is important in order * to minimize the possible polling on the hardware interconnect * by a remote user of this lock. * 2. Make the hwspinlock SMP-safe (so we can take it from * additional contexts on the local host). * 3. Ensure that in_atomic/might_sleep checks catch potential * problems with hwspinlock usage (e.g. scheduler checks like * 'scheduling while atomic' etc.) */ if (mode == HWLOCK_IRQSTATE) ret = spin_trylock_irqsave(&hwlock->lock, *flags); else if (mode == HWLOCK_IRQ) ret = spin_trylock_irq(&hwlock->lock); else ret = spin_trylock(&hwlock->lock); /* is lock already taken by another context on the local cpu ? */ if (!ret) return -EBUSY; /* try to take the hwspinlock device */ ret = hwlock->bank->ops->trylock(hwlock); /* if hwlock is already taken, undo spin_trylock_* and exit */ if (!ret) { if (mode == HWLOCK_IRQSTATE) spin_unlock_irqrestore(&hwlock->lock, *flags); else if (mode == HWLOCK_IRQ) spin_unlock_irq(&hwlock->lock); else spin_unlock(&hwlock->lock); return -EBUSY; } /* * We can be sure the other core's memory operations * are observable to us only _after_ we successfully take * the hwspinlock, and we must make sure that subsequent memory * operations (both reads and writes) will not be reordered before * we actually took the hwspinlock. * * Note: the implicit memory barrier of the spinlock above is too * early, so we need this additional explicit memory barrier. */ mb(); return 0; }
/** * x86_64 specific code for carrying out inter-CPU function calls. * This function should not be called directly. Call xcall_function() instead. * * Arguments: * [IN] cpu_mask: The target CPUs of the cross-call. * [IN] func: The function to execute on each target CPU. * [IN] info: Argument to pass to func(). * [IN] wait: true = wait for cross-call to fully complete. * * Returns: * Success: 0 * Failure: Error code */ int arch_xcall_function( cpumask_t cpu_mask, void (*func)(void *info), void * info, bool wait ) { struct xcall_data_struct data; unsigned int num_cpus; unsigned int cpu; BUG_ON(irqs_disabled()); /* Count how many CPUs are being targeted */ num_cpus = cpus_weight(cpu_mask); if (!num_cpus) return 0; /* Fill in the xcall data structure on our stack */ data.func = func; data.info = info; atomic_set(&data.started, 0); if (wait) atomic_set(&data.finished, 0); data.wait = wait; /* Spin with IRQs enabled */ while (!spin_trylock_irq(&xcall_data_lock)) ; /* IRQs are now disabled */ /* Set the global xcall data pointer */ xcall_data = &data; wmb(); /* Send inter-processor interrupts to the target CPUs */ for_each_cpu_mask(cpu, cpu_mask) lapic_send_ipi(cpu, XCALL_FUNCTION_VECTOR); /* Wait for initiation responses */ while (atomic_read(&data.started) != num_cpus) cpu_relax(); /* If requested, wait for completion responses */ if (wait) { while (atomic_read(&data.finished) != num_cpus) cpu_relax(); } spin_unlock_irq(&xcall_data_lock); return 0; }
void mlx4_en_poll_tx_cq(unsigned long data) { struct mlx4_en_cq *cq = (struct mlx4_en_cq *) data; struct mlx4_en_priv *priv = netdev_priv(cq->dev); struct mlx4_en_tx_ring *ring = &priv->tx_ring[cq->ring]; u32 inflight; INC_PERF_COUNTER(priv->pstats.tx_poll); if (!spin_trylock_irq(&ring->comp_lock)) { mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT); return; } mlx4_en_process_tx_cq(cq->dev, cq); inflight = (u32) (ring->prod - ring->cons - ring->last_nr_txbb); /* If there are still packets in flight and the timer has not already * been scheduled by the Tx routine then schedule it here to guarantee * completion processing of these packets */ if (inflight && priv->port_up) mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT); spin_unlock_irq(&ring->comp_lock); }
void mlx4_en_poll_tx_cq(unsigned long data) { struct mlx4_en_cq *cq = (struct mlx4_en_cq *) data; struct mlx4_en_priv *priv = netdev_priv(cq->dev); struct mlx4_en_tx_ring *ring = &priv->tx_ring[cq->ring]; u32 inflight; INC_PERF_COUNTER(priv->pstats.tx_poll); if (!spin_trylock_irq(&ring->comp_lock)) { mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT); return; } mlx4_en_process_tx_cq(cq->dev, cq); inflight = (u32) (ring->prod - ring->cons - ring->last_nr_txbb); /* */ if (inflight && priv->port_up) mod_timer(&cq->timer, jiffies + MLX4_EN_TX_POLL_TIMEOUT); spin_unlock_irq(&ring->comp_lock); }