/* Interrupts must be disabled (for the fallback code to work right) */ static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page, void *freelist_old, unsigned long counters_old, void *freelist_new, unsigned long counters_new, const char *n) { VM_BUG_ON(!irqs_disabled()); #ifdef CONFIG_CMPXCHG_DOUBLE if (s->flags & __CMPXCHG_DOUBLE) { if (cmpxchg_double(&page->freelist, freelist_old, counters_old, freelist_new, counters_new)) return 1; } else #endif { slab_lock(page); if (page->freelist == freelist_old && page->counters == counters_old) { page->freelist = freelist_new; page->counters = counters_new; slab_unlock(page); return 1; } slab_unlock(page); } cpu_relax(); stat(s, CMPXCHG_DOUBLE_FAIL); #ifdef SLUB_DEBUG_CMPXCHG printk(KERN_INFO "%s %s: cmpxchg double redo ", n, s->name); #endif return 0; }
static int modify_irte(struct irq_2_iommu *irq_iommu, struct irte *irte_modified) { struct intel_iommu *iommu; unsigned long flags; struct irte *irte; int rc, index; if (!irq_iommu) return -1; raw_spin_lock_irqsave(&irq_2_ir_lock, flags); iommu = irq_iommu->iommu; index = irq_iommu->irte_index + irq_iommu->sub_handle; irte = &iommu->ir_table->base[index]; #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) if ((irte->pst == 1) || (irte_modified->pst == 1)) { bool ret; ret = cmpxchg_double(&irte->low, &irte->high, irte->low, irte->high, irte_modified->low, irte_modified->high); /* * We use cmpxchg16 to atomically update the 128-bit IRTE, * and it cannot be updated by the hardware or other processors * behind us, so the return value of cmpxchg16 should be the * same as the old value. */ WARN_ON(!ret); } else #endif { set_64bit(&irte->low, irte_modified->low); set_64bit(&irte->high, irte_modified->high); } __iommu_flush_cache(iommu, irte, sizeof(*irte)); rc = qi_flush_iec(iommu, index, 0); /* Update iommu mode according to the IRTE mode */ irq_iommu->mode = irte->pst ? IRQ_POSTING : IRQ_REMAPPING; raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); return rc; }