void fscrypt_put_encryption_info(struct inode *inode, struct fscrypt_info *ci)
{
struct fscrypt_info *prev;
if (ci == NULL)
ci = ACCESS_ONCE(inode->i_crypt_info);
if (ci == NULL)
return;
prev = cmpxchg(&inode->i_crypt_info, ci, NULL);
if (prev != ci)
return;
put_crypt_info(ci);
}
void f2fs_free_encryption_info(struct inode *inode, struct f2fs_crypt_info *ci)
{
struct f2fs_inode_info *fi = F2FS_I(inode);
struct f2fs_crypt_info *prev;
if (ci == NULL)
ci = ACCESS_ONCE(fi->i_crypt_info);
if (ci == NULL)
return;
prev = cmpxchg(&fi->i_crypt_info, ci, NULL);
if (prev != ci)
return;
f2fs_free_crypt_info(ci);
}
int page_nid_xchg_last(struct page *page, int nid)
{
unsigned long old_flags, flags;
int last_nid;
do {
old_flags = flags = page->flags;
last_nid = page_nid_last(page);
flags &= ~(LAST_NID_MASK << LAST_NID_PGSHIFT);
flags |= (nid & LAST_NID_MASK) << LAST_NID_PGSHIFT;
} while (unlikely(cmpxchg(&page->flags, old_flags, flags) != old_flags));
return last_nid;
}
static void net_secret_init(void)
{
u32 tmp;
int i;
if (likely(net_secret[0]))
return;
for (i = NET_SECRET_SIZE; i > 0;) {
do {
get_random_bytes(&tmp, sizeof(tmp));
} while (!tmp);
cmpxchg(&net_secret[--i], 0, tmp);
}
}
/*
* FIXME: this is hack to save delta to linux buffer_head.
* Inefficient, and this is not atomic with dirty bit change. And this
* may not work on all arch (If set_bit() and cmpxchg() is not
* exclusive, this has race).
*/
static void tux3_set_bufdelta(struct buffer_head *buffer, int delta)
{
unsigned long state, old_state;
delta = tux3_delta(delta);
state = buffer->b_state;
for (;;) {
old_state = state;
state = tux3_bufsta_update(old_state, delta);
state = cmpxchg(&buffer->b_state, old_state, state);
if (state == old_state)
break;
}
}
void mce_log(struct mce *mce)
{
unsigned next, entry;
int ret = 0;
/* Emit the trace record: */
trace_mce_record(mce);
ret = atomic_notifier_call_chain(&x86_mce_decoder_chain, 0, mce);
if (ret == NOTIFY_STOP)
return;
mce->finished = 0;
wmb();
for (;;) {
entry = rcu_dereference_check_mce(mcelog.next);
for (;;) {
/*
* When the buffer fills up discard new entries.
* Assume that the earlier errors are the more
* interesting ones:
*/
if (entry >= MCE_LOG_LEN) {
set_bit(MCE_OVERFLOW,
(unsigned long *)&mcelog.flags);
return;
}
/* Old left over entry. Skip: */
if (mcelog.entry[entry].finished) {
entry++;
continue;
}
break;
}
smp_rmb();
next = entry + 1;
if (cmpxchg(&mcelog.next, entry, next) == entry)
break;
}
memcpy(mcelog.entry + entry, mce, sizeof(struct mce));
wmb();
mcelog.entry[entry].finished = 1;
wmb();
mce->finished = 1;
set_bit(0, &mce_need_notify);
}
void ext4_free_encryption_info(struct inode *inode,
struct ext4_crypt_info *ci)
{
struct ext4_inode_info *ei = EXT4_I(inode);
struct ext4_crypt_info *prev;
if (ci == NULL)
ci = ACCESS_ONCE(ei->i_crypt_info);
if (ci == NULL)
return;
prev = cmpxchg(&ei->i_crypt_info, ci, NULL);
if (prev != ci)
return;
ext4_free_crypt_info(ci);
}
static void check_lock(struct lock_debug *debug)
{
int irq_safe = !local_irq_is_enabled();
if ( unlikely(atomic_read(&spin_debug) <= 0) )
return;
/* A few places take liberties with this. */
/* BUG_ON(in_irq() && !irq_safe); */
if ( unlikely(debug->irq_safe != irq_safe) )
{
int seen = cmpxchg(&debug->irq_safe, -1, irq_safe);
BUG_ON(seen == !irq_safe);
}
}
int bbgl_mutex_lock(bbgl_mutex_t *mutex) {
int c;
for (int i = 0; i < 100; i++) {
c = cmpxchg(mutex, 0, 1);
if (c == 0)
return 0;
__asm__ __volatile__("pause" ::: "memory");
}
if (c == 1)
c = xchg(mutex, 2);
while (c) {
futex(mutex, FUTEX_WAIT, 2, NULL, NULL, 0);
c = xchg(mutex, 2);
}
return 0;
}
static void j2_send_ipi(unsigned int cpu, unsigned int message)
{
volatile unsigned *pmsg;
unsigned old;
unsigned long val;
/* There is only one IPI interrupt shared by all messages, so
* we keep a separate interrupt flag per message type in sw. */
pmsg = &per_cpu(j2_ipi_messages, cpu);
do old = *pmsg;
while (cmpxchg(pmsg, old, old|(1U<<message)) != old);
/* Generate the actual interrupt by writing to CCRn bit 28. */
val = __raw_readl(j2_ipi_trigger + cpu);
__raw_writel(val | (1U<<28), j2_ipi_trigger + cpu);
}
int gre_del_protocol(const struct gre_protocol *proto, u8 version)
{
int ret;
if (version >= GREPROTO_MAX)
return -EINVAL;
ret = (cmpxchg((const struct gre_protocol **)&gre_proto[version], proto, NULL) == proto) ?
0 : -EBUSY;
if (ret)
return ret;
synchronize_rcu();
return 0;
}
/*
* Claim the entry so that no one else will poke at it.
*/
static bool irq_work_claim(struct irq_work *work)
{
unsigned long flags, nflags;
for (;;) {
flags = work->flags;
if (flags & IRQ_WORK_PENDING)
return false;
nflags = flags | IRQ_WORK_FLAGS;
if (cmpxchg(&work->flags, flags, nflags) == flags)
break;
cpu_relax();
}
return true;
}
/*! 2017. 6. 3 study -ing */
int
task_work_add(struct task_struct *task, struct callback_head *work, bool notify)
{
struct callback_head *head;
do {
head = ACCESS_ONCE(task->task_works);
if (unlikely(head == &work_exited))
return -ESRCH;
work->next = head;
} while (cmpxchg(&task->task_works, head, work) != head);
if (notify)
set_notify_resume(task);
return 0;
}
unsigned long long
uisqueue_interlocked_and(unsigned long long __iomem *tgt,
unsigned long long set)
{
unsigned long long i;
unsigned long long j;
j = readq(tgt);
do {
i = j;
j = cmpxchg((__force unsigned long long *)tgt, i, i & set);
} while (i != j);
return j;
}
int ip_tunnel_encap_del_ops(const struct ip_tunnel_encap_ops *ops,
unsigned int num)
{
int ret;
if (num >= MAX_IPTUN_ENCAP_OPS)
return -ERANGE;
ret = (cmpxchg((const struct ip_tunnel_encap_ops **)
&iptun_encaps[num],
ops, NULL) == ops) ? 0 : -1;
synchronize_net();
return ret;
}
/**
* llist_add_batch - add several linked entries in batch
* @new_first: first entry in batch to be added
* @new_last: last entry in batch to be added
* @head: the head for your lock-less list
*
* Return whether list is empty before adding.
*/
bool llist_add_batch(struct llist_node *new_first, struct llist_node *new_last,
struct llist_head *head)
{
struct llist_node *entry, *old_entry;
entry = head->first;
for (;;) {
old_entry = entry;
new_last->next = entry;
entry = cmpxchg(&head->first, old_entry, new_first);
if (entry == old_entry)
break;
}
return old_entry == NULL;
}
static u16 nvmet_install_queue(struct nvmet_ctrl *ctrl, struct nvmet_req *req)
{
struct nvmf_connect_command *c = &req->cmd->connect;
u16 qid = le16_to_cpu(c->qid);
u16 sqsize = le16_to_cpu(c->sqsize);
struct nvmet_ctrl *old;
old = cmpxchg(&req->sq->ctrl, NULL, ctrl);
if (old) {
pr_warn("queue already connected!\n");
return NVME_SC_CONNECT_CTRL_BUSY | NVME_SC_DNR;
}
nvmet_cq_setup(ctrl, req->cq, qid, sqsize);
nvmet_sq_setup(ctrl, req->sq, qid, sqsize);
return 0;
}
static void amd_put_event_constraints(struct cpu_hw_events *cpuc,
struct perf_event *event)
{
struct hw_perf_event *hwc = &event->hw;
struct amd_nb *nb = cpuc->amd_nb;
int i;
if (!(amd_has_nb(cpuc) && amd_is_nb_event(hwc)))
return;
for (i = 0; i < x86_pmu.num_counters; i++) {
if (nb->owners[i] == event) {
cmpxchg(nb->owners+i, event, NULL);
break;
}
}
}
/**
* llist_del_first - delete the first entry of lock-less list
* @head: the head for your lock-less list
*
* If list is empty, return NULL, otherwise, return the first entry
* deleted, this is the newest added one.
*
* Only one llist_del_first user can be used simultaneously with
* multiple llist_add users without lock. Because otherwise
* llist_del_first, llist_add, llist_add (or llist_del_all, llist_add,
* llist_add) sequence in another user may change @head->first->next,
* but keep @head->first. If multiple consumers are needed, please
* use llist_del_all or use lock between consumers.
*/
struct llist_node *llist_del_first(struct llist_head *head)
{
struct llist_node *entry, *old_entry, *next;
entry = head->first;
for (;;) {
if (entry == NULL)
return NULL;
old_entry = entry;
next = entry->next;
entry = cmpxchg(&head->first, old_entry, next);
if (entry == old_entry)
break;
}
return entry;
}
static inline int alloc_descs(unsigned int start, unsigned int cnt, int node)
{
#if defined(CONFIG_KSTAT_IRQS_ONDEMAND)
struct irq_desc *desc;
unsigned int i;
for (i = 0; i < cnt; i++) {
desc = irq_to_desc(start + i);
if (desc && !desc->kstat_irqs) {
unsigned int __percpu *stats = alloc_percpu(unsigned int);
if (!stats)
return -1;
if (cmpxchg(&desc->kstat_irqs, NULL, stats) != NULL)
free_percpu(stats);
}
}
static irqreturn_t j2_ipi_interrupt_handler(int irq, void *arg)
{
unsigned cpu = hard_smp_processor_id();
volatile unsigned *pmsg = &per_cpu(j2_ipi_messages, cpu);
unsigned messages, i;
do messages = *pmsg;
while (cmpxchg(pmsg, messages, 0) != messages);
if (!messages) return IRQ_NONE;
for (i=0; i<SMP_MSG_NR; i++)
if (messages & (1U<<i))
smp_message_recv(i);
return IRQ_HANDLED;
}
int lwtunnel_encap_del_ops(const struct lwtunnel_encap_ops *ops,
unsigned int encap_type)
{
int ret;
if (encap_type == LWTUNNEL_ENCAP_NONE ||
encap_type > LWTUNNEL_ENCAP_MAX)
return -ERANGE;
ret = (cmpxchg((const struct lwtunnel_encap_ops **)
&lwtun_encaps[encap_type],
ops, NULL) == ops) ? 0 : -1;
synchronize_net();
return ret;
}
static void mspin_unlock(struct mspin_node **lock, struct mspin_node *node)
{
struct mspin_node *next = ACCESS_ONCE(node->next);
if (likely(!next)) {
/*
* Release the lock by setting it to NULL
*/
if (cmpxchg(lock, node, NULL) == node)
return;
/* Wait until the next pointer is set */
while (!(next = cpu_relaxed_read_long(&(node->next))))
cpu_read_relax();
}
ACCESS_ONCE(next->locked) = 1;
smp_wmb();
}
static void __irq_work_run(void)
{
unsigned long flags;
struct irq_work *work;
struct llist_head *this_list;
struct llist_node *llnode;
/*
* Reset the "raised" state right before we check the list because
* an NMI may enqueue after we find the list empty from the runner.
*/
__this_cpu_write(irq_work_raised, 0);
barrier();
this_list = &__get_cpu_var(irq_work_list);
if (llist_empty_relaxed(this_list))
return;
BUG_ON(!irqs_disabled());
llnode = llist_del_all(this_list);
while (llnode != NULL) {
work = llist_entry(llnode, struct irq_work, llnode);
llnode = llist_next(llnode);
/*
* Clear the PENDING bit, after this point the @work
* can be re-used.
* Make it immediately visible so that other CPUs trying
* to claim that work don't rely on us to handle their data
* while we are in the middle of the func.
*/
flags = work->flags & ~IRQ_WORK_PENDING;
xchg(&work->flags, flags);
work->func(work);
/*
* Clear the BUSY bit and return to the free state if
* no-one else claimed it meanwhile.
*/
(void)cmpxchg(&work->flags, flags, flags & ~IRQ_WORK_BUSY);
}
}
/*
* Queue the entry and raise the IPI if needed.
*/
static void __irq_work_queue(struct irq_work *entry)
{
struct irq_work **head, *next;
head = &get_cpu_var(irq_work_list);
do {
next = *head;
/* Can assign non-atomic because we keep the flags set. */
entry->next = next_flags(next, IRQ_WORK_FLAGS);
} while (cmpxchg(head, next, entry) != next);
/* The list was empty, raise self-interrupt to start processing. */
if (!irq_work_next(entry))
arch_irq_work_raise();
put_cpu_var(irq_work_list);
}
static void mspin_unlock(mspin_lock_t *lock, mspin_node_t *node)
{
mspin_node_t *next = ACCESS_ONCE(node->next);
if (likely(!next)) {
/*
* Release the lock by setting it to NULL
*/
if (cmpxchg(lock, node, NULL) == node)
return;
/* Wait until the next pointer is set */
while (!(next = ACCESS_ONCE(node->next)))
arch_mutex_cpu_relax();
}
barrier();
ACCESS_ONCE(next->locked) = 1;
smp_wmb();
}
static void catchup_quanta(quanta_t from, quanta_t target,
struct pfair_state* state)
{
quanta_t cur = from, time;
TRACE("+++< BAD catching up quanta from %lu to %lu\n",
from, target);
while (time_before(cur, target)) {
wait_for_quantum(cur, state);
cur++;
time = cmpxchg(&pfair_time,
cur - 1, /* expected */
cur /* next */
);
if (time == cur - 1)
schedule_next_quantum(cur);
}
TRACE("+++> catching up done\n");
}
void *ring_pop(struct ring_buf *ring)
{
void *elem, **slot;
unsigned int front;
for (;;) {
front = ring_front(ring);
if (front == ring_back(ring))
return NULL;
slot = ring->data + (front % ring->length);
if (!(elem = *slot))
continue;
if (cmpxchg(slot, elem, NULL) == elem)
break;
}
atomic_inc(&ring->front);
return elem;
}
/**
* task_work_run - execute the works added by task_work_add()
*
* Flush the pending works. Should be used by the core kernel code.
* Called before the task returns to the user-mode or stops, or when
* it exits. In the latter case task_work_add() can no longer add the
* new work after task_work_run() returns.
*/
void task_work_run(void)
{
struct task_struct *task = current;
struct callback_head *work, *head, *next;
for (;;) {
/*
* work->func() can do task_work_add(), do not set
* work_exited unless the list is empty.
*/
do {
work = ACCESS_ONCE(task->task_works);
head = !work && (task->flags & PF_EXITING) ?
&work_exited : NULL;
} while (cmpxchg(&task->task_works, work, head) != work);
if (!work)
break;
/*
* Synchronize with task_work_cancel(). It can't remove
* the first entry == work, cmpxchg(task_works) should
* fail, but it can play with *work and other entries.
*/
raw_spin_unlock_wait(&task->pi_lock);
smp_mb();
/* Reverse the list to run the works in fifo order */
head = NULL;
do {
next = work->next;
work->next = head;
head = work;
work = next;
} while (work);
work = head;
do {
next = work->next;
work->func(work);
work = next;
cond_resched();
} while (work);
}
}
void *SharedMemory::lock()
{
if(rptr==MAP_FAILED)return 0;
int c;
c = cmpxchg(&SH_MUTEX(rptr), 0, 1);
if(!c)return (char*)rptr+sizeof(SegmentHeader);
if(c==1)c=xchg(&SH_MUTEX(rptr), 2);
while(c)
{
sys_futex(&SH_MUTEX(rptr), FUTEX_WAIT, 2, NULL, NULL, 0);
c = xchg(&SH_MUTEX(rptr), 2);
}
return (char*)rptr+sizeof(SegmentHeader);
}
