int __intel_context_do_pin(struct intel_context *ce) { int err; if (mutex_lock_interruptible(&ce->pin_mutex)) return -EINTR; if (likely(!atomic_read(&ce->pin_count))) { intel_wakeref_t wakeref; err = 0; with_intel_runtime_pm(ce->engine->i915, wakeref) err = ce->ops->pin(ce); if (err) goto err; i915_gem_context_get(ce->gem_context); /* for ctx->ppgtt */ intel_context_get(ce); smp_mb__before_atomic(); /* flush pin before it is visible */ } atomic_inc(&ce->pin_count); GEM_BUG_ON(!intel_context_is_pinned(ce)); /* no overflow! */ mutex_unlock(&ce->pin_mutex); return 0; err: mutex_unlock(&ce->pin_mutex); return err; }
static void coupled_barrier(atomic_t *a, unsigned online) { /* * This function is effectively the same as * cpuidle_coupled_parallel_barrier, which can't be used here since * there's no cpuidle device. */ if (!coupled_coherence) return; smp_mb__before_atomic(); atomic_inc(a); while (atomic_read(a) < online) cpu_relax(); if (atomic_inc_return(a) == online * 2) { atomic_set(a, 0); return; } while (atomic_read(a) > online) cpu_relax(); }
static void mdp5_hw_unmask_irq(struct irq_data *irqd) { struct mdp5_kms *mdp5_kms = irq_data_get_irq_chip_data(irqd); smp_mb__before_atomic(); set_bit(irqd->hwirq, &mdp5_kms->irqcontroller.enabled_mask); smp_mb__after_atomic(); }
static void mdss_hw_mask_irq(struct irq_data *irqd) { struct msm_mdss *mdss = irq_data_get_irq_chip_data(irqd); smp_mb__before_atomic(); clear_bit(irqd->hwirq, &mdss->irqcontroller.enabled_mask); smp_mb__after_atomic(); }
static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter) { BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state)); /* flush memory to make sure state is correct before next watchdog */ smp_mb__before_atomic(); clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state); }
/** * nfs_unlock_request - Unlock request and wake up sleepers. * @req: */ void nfs_unlock_request(struct nfs_page *req) { if (!NFS_WBACK_BUSY(req)) { printk(KERN_ERR "NFS: Invalid unlock attempted\n"); BUG(); } smp_mb__before_atomic(); clear_bit(PG_BUSY, &req->wb_flags); smp_mb__after_atomic(); wake_up_bit(&req->wb_flags, PG_BUSY); }
/* * nfs_page_group_unlock - unlock the head of the page group * @req - request in group that is to be unlocked */ void nfs_page_group_unlock(struct nfs_page *req) { struct nfs_page *head = req->wb_head; WARN_ON_ONCE(head != head->wb_head); smp_mb__before_atomic(); clear_bit(PG_HEADLOCK, &head->wb_flags); smp_mb__after_atomic(); wake_up_bit(&head->wb_flags, PG_HEADLOCK); }
/* gss_cred_set_ctx: * called by gss_upcall_callback and gss_create_upcall in order * to set the gss context. The actual exchange of an old context * and a new one is protected by the pipe->lock. */ static void gss_cred_set_ctx(struct rpc_cred *cred, struct gss_cl_ctx *ctx) { struct gss_cred *gss_cred = container_of(cred, struct gss_cred, gc_base); if (!test_bit(RPCAUTH_CRED_NEW, &cred->cr_flags)) return; gss_get_ctx(ctx); rcu_assign_pointer(gss_cred->gc_ctx, ctx); set_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags); smp_mb__before_atomic(); clear_bit(RPCAUTH_CRED_NEW, &cred->cr_flags); }
static inline void seccomp_assign_mode(struct task_struct *task, unsigned long seccomp_mode) { assert_spin_locked(&task->sighand->siglock); task->seccomp.mode = seccomp_mode; /* * Make sure TIF_SECCOMP cannot be set before the mode (and * filter) is set. */ smp_mb__before_atomic(); set_tsk_thread_flag(task, TIF_SECCOMP); }
/* sndbuf consumer */ static inline void smc_tx_advance_cursors(struct smc_connection *conn, union smc_host_cursor *prod, union smc_host_cursor *sent, size_t len) { smc_curs_add(conn->peer_rmbe_size, prod, len); /* increased in recv tasklet smc_cdc_msg_rcv() */ smp_mb__before_atomic(); /* data in flight reduces usable snd_wnd */ atomic_sub(len, &conn->peer_rmbe_space); /* guarantee 0 <= peer_rmbe_space <= peer_rmbe_size */ smp_mb__after_atomic(); smc_curs_add(conn->sndbuf_size, sent, len); }
static inline void seccomp_assign_mode(struct task_struct *task, unsigned long seccomp_mode, unsigned long flags) { assert_spin_locked(&task->sighand->siglock); task->seccomp.mode = seccomp_mode; /* * Make sure TIF_SECCOMP cannot be set before the mode (and * filter) is set. */ smp_mb__before_atomic(); /* Assume default seccomp processes want spec flaw mitigation. */ if ((flags & SECCOMP_FILTER_FLAG_SPEC_ALLOW) == 0) arch_seccomp_spec_mitigate(task); set_tsk_thread_flag(task, TIF_SECCOMP); }
void sbitmap_queue_resize(struct sbitmap_queue *sbq, unsigned int depth) { unsigned int wake_batch = sbq_calc_wake_batch(depth); int i; if (sbq->wake_batch != wake_batch) { WRITE_ONCE(sbq->wake_batch, wake_batch); /* * Pairs with the memory barrier in sbq_wake_up() to ensure that * the batch size is updated before the wait counts. */ smp_mb__before_atomic(); for (i = 0; i < SBQ_WAIT_QUEUES; i++) atomic_set(&sbq->ws[i].wait_cnt, 1); } sbitmap_resize(&sbq->sb, depth); }
static int async_chainiv_schedule_work(struct async_chainiv_ctx *ctx) { int queued; int err = ctx->err; if (!ctx->queue.qlen) { smp_mb__before_atomic(); clear_bit(CHAINIV_STATE_INUSE, &ctx->state); if (!ctx->queue.qlen || test_and_set_bit(CHAINIV_STATE_INUSE, &ctx->state)) goto out; } queued = queue_work(kcrypto_wq, &ctx->postponed); BUG_ON(!queued); out: return err; }
static void sbq_wake_up(struct sbitmap_queue *sbq) { struct sbq_wait_state *ws; unsigned int wake_batch; int wait_cnt; /* * Pairs with the memory barrier in set_current_state() to ensure the * proper ordering of clear_bit()/waitqueue_active() in the waker and * test_and_set_bit()/prepare_to_wait()/finish_wait() in the waiter. See * the comment on waitqueue_active(). This is __after_atomic because we * just did clear_bit() in the caller. */ smp_mb__after_atomic(); ws = sbq_wake_ptr(sbq); if (!ws) return; wait_cnt = atomic_dec_return(&ws->wait_cnt); if (wait_cnt <= 0) { wake_batch = READ_ONCE(sbq->wake_batch); /* * Pairs with the memory barrier in sbitmap_queue_resize() to * ensure that we see the batch size update before the wait * count is reset. */ smp_mb__before_atomic(); /* * If there are concurrent callers to sbq_wake_up(), the last * one to decrement the wait count below zero will bump it back * up. If there is a concurrent resize, the count reset will * either cause the cmpxchg to fail or overwrite after the * cmpxchg. */ atomic_cmpxchg(&ws->wait_cnt, wait_cnt, wait_cnt + wake_batch); sbq_index_atomic_inc(&sbq->wake_index); wake_up(&ws->wait); } }
static void inode_go_sync(struct gfs2_glock *gl) { struct gfs2_inode *ip = gfs2_glock2inode(gl); int isreg = ip && S_ISREG(ip->i_inode.i_mode); struct address_space *metamapping = gfs2_glock2aspace(gl); int error; if (isreg) { if (test_and_clear_bit(GIF_SW_PAGED, &ip->i_flags)) unmap_shared_mapping_range(ip->i_inode.i_mapping, 0, 0); inode_dio_wait(&ip->i_inode); } if (!test_and_clear_bit(GLF_DIRTY, &gl->gl_flags)) goto out; GLOCK_BUG_ON(gl, gl->gl_state != LM_ST_EXCLUSIVE); gfs2_log_flush(gl->gl_name.ln_sbd, gl, GFS2_LOG_HEAD_FLUSH_NORMAL | GFS2_LFC_INODE_GO_SYNC); filemap_fdatawrite(metamapping); if (isreg) { struct address_space *mapping = ip->i_inode.i_mapping; filemap_fdatawrite(mapping); error = filemap_fdatawait(mapping); mapping_set_error(mapping, error); } error = filemap_fdatawait(metamapping); mapping_set_error(metamapping, error); gfs2_ail_empty_gl(gl); /* * Writeback of the data mapping may cause the dirty flag to be set * so we have to clear it again here. */ smp_mb__before_atomic(); clear_bit(GLF_DIRTY, &gl->gl_flags); out: gfs2_clear_glop_pending(ip); }
static void linkwatch_do_dev(struct net_device *dev) { /* * Make sure the above read is complete since it can be * rewritten as soon as we clear the bit below. */ smp_mb__before_atomic(); /* We are about to handle this device, * so new events can be accepted */ clear_bit(__LINK_STATE_LINKWATCH_PENDING, &dev->state); rfc2863_policy(dev); if (dev->flags & IFF_UP && netif_device_present(dev)) { if (netif_carrier_ok(dev)) dev_activate(dev); else dev_deactivate(dev); netdev_state_change(dev); } dev_put(dev); }
static void inode_go_sync(struct gfs2_glock *gl) { struct gfs2_inode *ip = gl->gl_object; struct address_space *metamapping = gfs2_glock2aspace(gl); int error; if (ip && !S_ISREG(ip->i_inode.i_mode)) ip = NULL; if (ip) { if (test_and_clear_bit(GIF_SW_PAGED, &ip->i_flags)) unmap_shared_mapping_range(ip->i_inode.i_mapping, 0, 0); inode_dio_wait(&ip->i_inode); } if (!test_and_clear_bit(GLF_DIRTY, &gl->gl_flags)) return; GLOCK_BUG_ON(gl, gl->gl_state != LM_ST_EXCLUSIVE); gfs2_log_flush(gl->gl_sbd, gl, NORMAL_FLUSH); filemap_fdatawrite(metamapping); if (ip) { struct address_space *mapping = ip->i_inode.i_mapping; filemap_fdatawrite(mapping); error = filemap_fdatawait(mapping); mapping_set_error(mapping, error); } error = filemap_fdatawait(metamapping); mapping_set_error(metamapping, error); gfs2_ail_empty_gl(gl); /* * Writeback of the data mapping may cause the dirty flag to be set * so we have to clear it again here. */ smp_mb__before_atomic(); clear_bit(GLF_DIRTY, &gl->gl_flags); }
static int _nfs42_proc_fallocate(struct rpc_message *msg, struct file *filep, loff_t offset, loff_t len) { struct inode *inode = file_inode(filep); struct nfs_server *server = NFS_SERVER(inode); struct nfs42_falloc_args args = { .falloc_fh = NFS_FH(inode), .falloc_offset = offset, .falloc_length = len, .falloc_bitmask = server->cache_consistency_bitmask, }; struct nfs42_falloc_res res = { .falloc_server = server, }; int status; msg->rpc_argp = &args; msg->rpc_resp = &res; status = nfs42_set_rw_stateid(&args.falloc_stateid, filep, FMODE_WRITE); if (status) return status; res.falloc_fattr = nfs_alloc_fattr(); if (!res.falloc_fattr) return -ENOMEM; status = nfs4_call_sync(server->client, server, msg, &args.seq_args, &res.seq_res, 0); if (status == 0) status = nfs_post_op_update_inode(inode, res.falloc_fattr); kfree(res.falloc_fattr); return status; } static int nfs42_proc_fallocate(struct rpc_message *msg, struct file *filep, loff_t offset, loff_t len) { struct nfs_server *server = NFS_SERVER(file_inode(filep)); struct nfs4_exception exception = { }; int err; do { err = _nfs42_proc_fallocate(msg, filep, offset, len); if (err == -ENOTSUPP) return -EOPNOTSUPP; err = nfs4_handle_exception(server, err, &exception); } while (exception.retry); return err; } int nfs42_proc_allocate(struct file *filep, loff_t offset, loff_t len) { struct rpc_message msg = { .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_ALLOCATE], }; struct inode *inode = file_inode(filep); int err; if (!nfs_server_capable(inode, NFS_CAP_ALLOCATE)) return -EOPNOTSUPP; mutex_lock(&inode->i_mutex); err = nfs42_proc_fallocate(&msg, filep, offset, len); if (err == -EOPNOTSUPP) NFS_SERVER(inode)->caps &= ~NFS_CAP_ALLOCATE; mutex_unlock(&inode->i_mutex); return err; } int nfs42_proc_deallocate(struct file *filep, loff_t offset, loff_t len) { struct rpc_message msg = { .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_DEALLOCATE], }; struct inode *inode = file_inode(filep); int err; if (!nfs_server_capable(inode, NFS_CAP_DEALLOCATE)) return -EOPNOTSUPP; nfs_wb_all(inode); mutex_lock(&inode->i_mutex); err = nfs42_proc_fallocate(&msg, filep, offset, len); if (err == 0) truncate_pagecache_range(inode, offset, (offset + len) -1); if (err == -EOPNOTSUPP) NFS_SERVER(inode)->caps &= ~NFS_CAP_DEALLOCATE; mutex_unlock(&inode->i_mutex); return err; } static loff_t _nfs42_proc_llseek(struct file *filep, loff_t offset, int whence) { struct inode *inode = file_inode(filep); struct nfs42_seek_args args = { .sa_fh = NFS_FH(inode), .sa_offset = offset, .sa_what = (whence == SEEK_HOLE) ? NFS4_CONTENT_HOLE : NFS4_CONTENT_DATA, }; struct nfs42_seek_res res; struct rpc_message msg = { .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_SEEK], .rpc_argp = &args, .rpc_resp = &res, }; struct nfs_server *server = NFS_SERVER(inode); int status; if (!nfs_server_capable(inode, NFS_CAP_SEEK)) return -ENOTSUPP; status = nfs42_set_rw_stateid(&args.sa_stateid, filep, FMODE_READ); if (status) return status; nfs_wb_all(inode); status = nfs4_call_sync(server->client, server, &msg, &args.seq_args, &res.seq_res, 0); if (status == -ENOTSUPP) server->caps &= ~NFS_CAP_SEEK; if (status) return status; return vfs_setpos(filep, res.sr_offset, inode->i_sb->s_maxbytes); } loff_t nfs42_proc_llseek(struct file *filep, loff_t offset, int whence) { struct nfs_server *server = NFS_SERVER(file_inode(filep)); struct nfs4_exception exception = { }; loff_t err; do { err = _nfs42_proc_llseek(filep, offset, whence); if (err >= 0) break; if (err == -ENOTSUPP) return -EOPNOTSUPP; err = nfs4_handle_exception(server, err, &exception); } while (exception.retry); return err; } static void nfs42_layoutstat_prepare(struct rpc_task *task, void *calldata) { struct nfs42_layoutstat_data *data = calldata; struct nfs_server *server = NFS_SERVER(data->args.inode); nfs41_setup_sequence(nfs4_get_session(server), &data->args.seq_args, &data->res.seq_res, task); } static void nfs42_layoutstat_done(struct rpc_task *task, void *calldata) { struct nfs42_layoutstat_data *data = calldata; if (!nfs4_sequence_done(task, &data->res.seq_res)) return; switch (task->tk_status) { case 0: break; case -ENOTSUPP: case -EOPNOTSUPP: NFS_SERVER(data->inode)->caps &= ~NFS_CAP_LAYOUTSTATS; default: dprintk("%s server returns %d\n", __func__, task->tk_status); } } static void nfs42_layoutstat_release(void *calldata) { struct nfs42_layoutstat_data *data = calldata; struct nfs_server *nfss = NFS_SERVER(data->args.inode); if (nfss->pnfs_curr_ld->cleanup_layoutstats) nfss->pnfs_curr_ld->cleanup_layoutstats(data); pnfs_put_layout_hdr(NFS_I(data->args.inode)->layout); smp_mb__before_atomic(); clear_bit(NFS_INO_LAYOUTSTATS, &NFS_I(data->args.inode)->flags); smp_mb__after_atomic(); nfs_iput_and_deactive(data->inode); kfree(data->args.devinfo); kfree(data); } static const struct rpc_call_ops nfs42_layoutstat_ops = { .rpc_call_prepare = nfs42_layoutstat_prepare, .rpc_call_done = nfs42_layoutstat_done, .rpc_release = nfs42_layoutstat_release, }; int nfs42_proc_layoutstats_generic(struct nfs_server *server, struct nfs42_layoutstat_data *data) { struct rpc_message msg = { .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_LAYOUTSTATS], .rpc_argp = &data->args, .rpc_resp = &data->res, }; struct rpc_task_setup task_setup = { .rpc_client = server->client, .rpc_message = &msg, .callback_ops = &nfs42_layoutstat_ops, .callback_data = data, .flags = RPC_TASK_ASYNC, }; struct rpc_task *task; data->inode = nfs_igrab_and_active(data->args.inode); if (!data->inode) { nfs42_layoutstat_release(data); return -EAGAIN; } nfs4_init_sequence(&data->args.seq_args, &data->res.seq_res, 0); task = rpc_run_task(&task_setup); if (IS_ERR(task)) return PTR_ERR(task); return 0; } static int _nfs42_proc_clone(struct rpc_message *msg, struct file *src_f, struct file *dst_f, loff_t src_offset, loff_t dst_offset, loff_t count) { struct inode *src_inode = file_inode(src_f); struct inode *dst_inode = file_inode(dst_f); struct nfs_server *server = NFS_SERVER(dst_inode); struct nfs42_clone_args args = { .src_fh = NFS_FH(src_inode), .dst_fh = NFS_FH(dst_inode), .src_offset = src_offset, .dst_offset = dst_offset, .count = count, .dst_bitmask = server->cache_consistency_bitmask, }; struct nfs42_clone_res res = { .server = server, }; int status; msg->rpc_argp = &args; msg->rpc_resp = &res; status = nfs42_set_rw_stateid(&args.src_stateid, src_f, FMODE_READ); if (status) return status; status = nfs42_set_rw_stateid(&args.dst_stateid, dst_f, FMODE_WRITE); if (status) return status; res.dst_fattr = nfs_alloc_fattr(); if (!res.dst_fattr) return -ENOMEM; status = nfs4_call_sync(server->client, server, msg, &args.seq_args, &res.seq_res, 0); if (status == 0) status = nfs_post_op_update_inode(dst_inode, res.dst_fattr); kfree(res.dst_fattr); return status; } int nfs42_proc_clone(struct file *src_f, struct file *dst_f, loff_t src_offset, loff_t dst_offset, loff_t count) { struct rpc_message msg = { .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_CLONE], }; struct inode *inode = file_inode(src_f); struct nfs_server *server = NFS_SERVER(file_inode(src_f)); struct nfs4_exception exception = { }; int err; if (!nfs_server_capable(inode, NFS_CAP_CLONE)) return -EOPNOTSUPP; do { err = _nfs42_proc_clone(&msg, src_f, dst_f, src_offset, dst_offset, count); if (err == -ENOTSUPP || err == -EOPNOTSUPP) { NFS_SERVER(inode)->caps &= ~NFS_CAP_CLONE; return -EOPNOTSUPP; } err = nfs4_handle_exception(server, err, &exception); } while (exception.retry); return err; }
void free_irqno(unsigned int irq) { smp_mb__before_atomic(); clear_bit(irq, irq_map); smp_mb__after_atomic(); }
/* * Increment chain count for the bio. Make sure the CHAIN flag update * is visible before the raised count. */ static inline void bio_inc_remaining(struct bio *bio) { bio_set_flag(bio, BIO_CHAIN); smp_mb__before_atomic(); atomic_inc(&bio->__bi_remaining); }
/** * cpupri_set - update the cpu priority setting * @cp: The cpupri context * @cpu: The target cpu * @newpri: The priority (INVALID-RT99) to assign to this CPU * * Note: Assumes cpu_rq(cpu)->lock is locked * * Returns: (void) */ void cpupri_set(struct cpupri *cp, int cpu, int newpri) { int *currpri = &cp->cpu_to_pri[cpu]; int oldpri = *currpri; int do_mb = 0; newpri = convert_prio(newpri); BUG_ON(newpri >= CPUPRI_NR_PRIORITIES); if (newpri == oldpri) return; /* * If the cpu was currently mapped to a different value, we * need to map it to the new value then remove the old value. * Note, we must add the new value first, otherwise we risk the * cpu being missed by the priority loop in cpupri_find. */ if (likely(newpri != CPUPRI_INVALID)) { struct cpupri_vec *vec = &cp->pri_to_cpu[newpri]; cpumask_set_cpu(cpu, vec->mask); /* * When adding a new vector, we update the mask first, * do a write memory barrier, and then update the count, to * make sure the vector is visible when count is set. */ smp_mb__before_atomic(); atomic_inc(&(vec)->count); do_mb = 1; } if (likely(oldpri != CPUPRI_INVALID)) { struct cpupri_vec *vec = &cp->pri_to_cpu[oldpri]; /* * Because the order of modification of the vec->count * is important, we must make sure that the update * of the new prio is seen before we decrement the * old prio. This makes sure that the loop sees * one or the other when we raise the priority of * the run queue. We don't care about when we lower the * priority, as that will trigger an rt pull anyway. * * We only need to do a memory barrier if we updated * the new priority vec. */ if (do_mb) smp_mb__after_atomic(); /* * When removing from the vector, we decrement the counter first * do a memory barrier and then clear the mask. */ atomic_dec(&(vec)->count); smp_mb__after_atomic(); cpumask_clear_cpu(cpu, vec->mask); } *currpri = newpri; }
/* the core send_sem serializes this with other xmit and shutdown */ static int rds_tcp_sendmsg(struct socket *sock, void *data, unsigned int len) { struct kvec vec = { .iov_base = data, .iov_len = len, }; struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL, }; return kernel_sendmsg(sock, &msg, &vec, 1, vec.iov_len); } /* the core send_sem serializes this with other xmit and shutdown */ int rds_tcp_xmit(struct rds_connection *conn, struct rds_message *rm, unsigned int hdr_off, unsigned int sg, unsigned int off) { struct rds_conn_path *cp = rm->m_inc.i_conn_path; struct rds_tcp_connection *tc = cp->cp_transport_data; int done = 0; int ret = 0; int more; if (hdr_off == 0) { /* * m_ack_seq is set to the sequence number of the last byte of * header and data. see rds_tcp_is_acked(). */ tc->t_last_sent_nxt = rds_tcp_snd_nxt(tc); rm->m_ack_seq = tc->t_last_sent_nxt + sizeof(struct rds_header) + be32_to_cpu(rm->m_inc.i_hdr.h_len) - 1; smp_mb__before_atomic(); set_bit(RDS_MSG_HAS_ACK_SEQ, &rm->m_flags); tc->t_last_expected_una = rm->m_ack_seq + 1; if (test_bit(RDS_MSG_RETRANSMITTED, &rm->m_flags)) rm->m_inc.i_hdr.h_flags |= RDS_FLAG_RETRANSMITTED; rdsdebug("rm %p tcp nxt %u ack_seq %llu\n", rm, rds_tcp_snd_nxt(tc), (unsigned long long)rm->m_ack_seq); } if (hdr_off < sizeof(struct rds_header)) { /* see rds_tcp_write_space() */ set_bit(SOCK_NOSPACE, &tc->t_sock->sk->sk_socket->flags); ret = rds_tcp_sendmsg(tc->t_sock, (void *)&rm->m_inc.i_hdr + hdr_off, sizeof(rm->m_inc.i_hdr) - hdr_off); if (ret < 0) goto out; done += ret; if (hdr_off + done != sizeof(struct rds_header)) goto out; } more = rm->data.op_nents > 1 ? (MSG_MORE | MSG_SENDPAGE_NOTLAST) : 0; while (sg < rm->data.op_nents) { int flags = MSG_DONTWAIT | MSG_NOSIGNAL | more; ret = tc->t_sock->ops->sendpage(tc->t_sock, sg_page(&rm->data.op_sg[sg]), rm->data.op_sg[sg].offset + off, rm->data.op_sg[sg].length - off, flags); rdsdebug("tcp sendpage %p:%u:%u ret %d\n", (void *)sg_page(&rm->data.op_sg[sg]), rm->data.op_sg[sg].offset + off, rm->data.op_sg[sg].length - off, ret); if (ret <= 0) break; off += ret; done += ret; if (off == rm->data.op_sg[sg].length) { off = 0; sg++; } if (sg == rm->data.op_nents - 1) more = 0; } out: if (ret <= 0) { /* write_space will hit after EAGAIN, all else fatal */ if (ret == -EAGAIN) { rds_tcp_stats_inc(s_tcp_sndbuf_full); ret = 0; } else { /* No need to disconnect/reconnect if path_drop * has already been triggered, because, e.g., of * an incoming RST. */ if (rds_conn_path_up(cp)) { pr_warn("RDS/tcp: send to %pI4 on cp [%d]" "returned %d, " "disconnecting and reconnecting\n", &conn->c_faddr, cp->cp_index, ret); rds_conn_path_drop(cp); } } } if (done == 0) done = ret; return done; } /* * rm->m_ack_seq is set to the tcp sequence number that corresponds to the * last byte of the message, including the header. This means that the * entire message has been received if rm->m_ack_seq is "before" the next * unacked byte of the TCP sequence space. We have to do very careful * wrapping 32bit comparisons here. */ static int rds_tcp_is_acked(struct rds_message *rm, uint64_t ack) { if (!test_bit(RDS_MSG_HAS_ACK_SEQ, &rm->m_flags)) return 0; return (__s32)((u32)rm->m_ack_seq - (u32)ack) < 0; } void rds_tcp_write_space(struct sock *sk) { void (*write_space)(struct sock *sk); struct rds_conn_path *cp; struct rds_tcp_connection *tc; read_lock_bh(&sk->sk_callback_lock); cp = sk->sk_user_data; if (!cp) { write_space = sk->sk_write_space; goto out; } tc = cp->cp_transport_data; rdsdebug("write_space for tc %p\n", tc); write_space = tc->t_orig_write_space; rds_tcp_stats_inc(s_tcp_write_space_calls); rdsdebug("tcp una %u\n", rds_tcp_snd_una(tc)); tc->t_last_seen_una = rds_tcp_snd_una(tc); rds_send_path_drop_acked(cp, rds_tcp_snd_una(tc), rds_tcp_is_acked); if ((atomic_read(&sk->sk_wmem_alloc) << 1) <= sk->sk_sndbuf) queue_delayed_work(rds_wq, &cp->cp_send_w, 0); out: read_unlock_bh(&sk->sk_callback_lock); /* * write_space is only called when data leaves tcp's send queue if * SOCK_NOSPACE is set. We set SOCK_NOSPACE every time we put * data in tcp's send queue because we use write_space to parse the * sequence numbers and notice that rds messages have been fully * received. * * tcp's write_space clears SOCK_NOSPACE if the send queue has more * than a certain amount of space. So we need to set it again *after* * we call tcp's write_space or else we might only get called on the * first of a series of incoming tcp acks. */ write_space(sk); if (sk->sk_socket) set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); }
/* sndbuf producer: main API called by socket layer. * called under sock lock. */ int smc_tx_sendmsg(struct smc_sock *smc, struct msghdr *msg, size_t len) { size_t copylen, send_done = 0, send_remaining = len; size_t chunk_len, chunk_off, chunk_len_sum; struct smc_connection *conn = &smc->conn; union smc_host_cursor prep; struct sock *sk = &smc->sk; char *sndbuf_base; int tx_cnt_prep; int writespace; int rc, chunk; /* This should be in poll */ sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) { rc = -EPIPE; goto out_err; } while (msg_data_left(msg)) { if (sk->sk_state == SMC_INIT) return -ENOTCONN; if (smc->sk.sk_shutdown & SEND_SHUTDOWN || (smc->sk.sk_err == ECONNABORTED) || conn->local_tx_ctrl.conn_state_flags.peer_conn_abort) return -EPIPE; if (smc_cdc_rxed_any_close(conn)) return send_done ?: -ECONNRESET; if (!atomic_read(&conn->sndbuf_space)) { rc = smc_tx_wait_memory(smc, msg->msg_flags); if (rc) { if (send_done) return send_done; goto out_err; } continue; } /* initialize variables for 1st iteration of subsequent loop */ /* could be just 1 byte, even after smc_tx_wait_memory above */ writespace = atomic_read(&conn->sndbuf_space); /* not more than what user space asked for */ copylen = min_t(size_t, send_remaining, writespace); /* determine start of sndbuf */ sndbuf_base = conn->sndbuf_desc->cpu_addr; smc_curs_write(&prep, smc_curs_read(&conn->tx_curs_prep, conn), conn); tx_cnt_prep = prep.count; /* determine chunks where to write into sndbuf */ /* either unwrapped case, or 1st chunk of wrapped case */ chunk_len = min_t(size_t, copylen, conn->sndbuf_size - tx_cnt_prep); chunk_len_sum = chunk_len; chunk_off = tx_cnt_prep; smc_sndbuf_sync_sg_for_cpu(conn); for (chunk = 0; chunk < 2; chunk++) { rc = memcpy_from_msg(sndbuf_base + chunk_off, msg, chunk_len); if (rc) { smc_sndbuf_sync_sg_for_device(conn); if (send_done) return send_done; goto out_err; } send_done += chunk_len; send_remaining -= chunk_len; if (chunk_len_sum == copylen) break; /* either on 1st or 2nd iteration */ /* prepare next (== 2nd) iteration */ chunk_len = copylen - chunk_len; /* remainder */ chunk_len_sum += chunk_len; chunk_off = 0; /* modulo offset in send ring buffer */ } smc_sndbuf_sync_sg_for_device(conn); /* update cursors */ smc_curs_add(conn->sndbuf_size, &prep, copylen); smc_curs_write(&conn->tx_curs_prep, smc_curs_read(&prep, conn), conn); /* increased in send tasklet smc_cdc_tx_handler() */ smp_mb__before_atomic(); atomic_sub(copylen, &conn->sndbuf_space); /* guarantee 0 <= sndbuf_space <= sndbuf_size */ smp_mb__after_atomic(); /* since we just produced more new data into sndbuf, * trigger sndbuf consumer: RDMA write into peer RMBE and CDC */ smc_tx_sndbuf_nonempty(conn); } /* while (msg_data_left(msg)) */ return send_done; out_err: rc = sk_stream_error(sk, msg->msg_flags, rc); /* make sure we wake any epoll edge trigger waiter */ if (unlikely(rc == -EAGAIN)) sk->sk_write_space(sk); return rc; }
static int kgdb_cpu_enter(struct kgdb_state *ks, struct pt_regs *regs, int exception_state) { unsigned long flags; int sstep_tries = 100; int error; int cpu; int trace_on = 0; int online_cpus = num_online_cpus(); u64 time_left; kgdb_info[ks->cpu].enter_kgdb++; kgdb_info[ks->cpu].exception_state |= exception_state; if (exception_state == DCPU_WANT_MASTER) atomic_inc(&masters_in_kgdb); else atomic_inc(&slaves_in_kgdb); if (arch_kgdb_ops.disable_hw_break) arch_kgdb_ops.disable_hw_break(regs); acquirelock: /* * Interrupts will be restored by the 'trap return' code, except when * single stepping. */ local_irq_save(flags); cpu = ks->cpu; kgdb_info[cpu].debuggerinfo = regs; kgdb_info[cpu].task = current; kgdb_info[cpu].ret_state = 0; kgdb_info[cpu].irq_depth = hardirq_count() >> HARDIRQ_SHIFT; /* Make sure the above info reaches the primary CPU */ smp_mb(); if (exception_level == 1) { if (raw_spin_trylock(&dbg_master_lock)) atomic_xchg(&kgdb_active, cpu); goto cpu_master_loop; } /* * CPU will loop if it is a slave or request to become a kgdb * master cpu and acquire the kgdb_active lock: */ while (1) { cpu_loop: if (kgdb_info[cpu].exception_state & DCPU_NEXT_MASTER) { kgdb_info[cpu].exception_state &= ~DCPU_NEXT_MASTER; goto cpu_master_loop; } else if (kgdb_info[cpu].exception_state & DCPU_WANT_MASTER) { if (raw_spin_trylock(&dbg_master_lock)) { atomic_xchg(&kgdb_active, cpu); break; } } else if (kgdb_info[cpu].exception_state & DCPU_IS_SLAVE) { if (!raw_spin_is_locked(&dbg_slave_lock)) goto return_normal; } else { return_normal: /* Return to normal operation by executing any * hw breakpoint fixup. */ if (arch_kgdb_ops.correct_hw_break) arch_kgdb_ops.correct_hw_break(); if (trace_on) tracing_on(); kgdb_info[cpu].exception_state &= ~(DCPU_WANT_MASTER | DCPU_IS_SLAVE); kgdb_info[cpu].enter_kgdb--; smp_mb__before_atomic(); atomic_dec(&slaves_in_kgdb); dbg_touch_watchdogs(); local_irq_restore(flags); return 0; } cpu_relax(); } /* * For single stepping, try to only enter on the processor * that was single stepping. To guard against a deadlock, the * kernel will only try for the value of sstep_tries before * giving up and continuing on. */ if (atomic_read(&kgdb_cpu_doing_single_step) != -1 && (kgdb_info[cpu].task && kgdb_info[cpu].task->pid != kgdb_sstep_pid) && --sstep_tries) { atomic_set(&kgdb_active, -1); raw_spin_unlock(&dbg_master_lock); dbg_touch_watchdogs(); local_irq_restore(flags); goto acquirelock; } if (!kgdb_io_ready(1)) { kgdb_info[cpu].ret_state = 1; goto kgdb_restore; /* No I/O connection, resume the system */ } /* * Don't enter if we have hit a removed breakpoint. */ if (kgdb_skipexception(ks->ex_vector, ks->linux_regs)) goto kgdb_restore; /* Call the I/O driver's pre_exception routine */ if (dbg_io_ops->pre_exception) dbg_io_ops->pre_exception(); /* * Get the passive CPU lock which will hold all the non-primary * CPU in a spin state while the debugger is active */ if (!kgdb_single_step) raw_spin_lock(&dbg_slave_lock); #ifdef CONFIG_SMP /* If send_ready set, slaves are already waiting */ if (ks->send_ready) atomic_set(ks->send_ready, 1); /* Signal the other CPUs to enter kgdb_wait() */ else if ((!kgdb_single_step) && kgdb_do_roundup) kgdb_roundup_cpus(flags); #endif /* * Wait for the other CPUs to be notified and be waiting for us: */ time_left = MSEC_PER_SEC; while (kgdb_do_roundup && --time_left && (atomic_read(&masters_in_kgdb) + atomic_read(&slaves_in_kgdb)) != online_cpus) udelay(1000); if (!time_left) pr_crit("Timed out waiting for secondary CPUs.\n"); /* * At this point the primary processor is completely * in the debugger and all secondary CPUs are quiescent */ dbg_deactivate_sw_breakpoints(); kgdb_single_step = 0; kgdb_contthread = current; exception_level = 0; trace_on = tracing_is_on(); if (trace_on) tracing_off(); while (1) { cpu_master_loop: if (dbg_kdb_mode) { kgdb_connected = 1; error = kdb_stub(ks); if (error == -1) continue; kgdb_connected = 0; } else { error = gdb_serial_stub(ks); } if (error == DBG_PASS_EVENT) { dbg_kdb_mode = !dbg_kdb_mode; } else if (error == DBG_SWITCH_CPU_EVENT) { kgdb_info[dbg_switch_cpu].exception_state |= DCPU_NEXT_MASTER; goto cpu_loop; } else { kgdb_info[cpu].ret_state = error; break; } } /* Call the I/O driver's post_exception routine */ if (dbg_io_ops->post_exception) dbg_io_ops->post_exception(); if (!kgdb_single_step) { raw_spin_unlock(&dbg_slave_lock); /* Wait till all the CPUs have quit from the debugger. */ while (kgdb_do_roundup && atomic_read(&slaves_in_kgdb)) cpu_relax(); } kgdb_restore: if (atomic_read(&kgdb_cpu_doing_single_step) != -1) { int sstep_cpu = atomic_read(&kgdb_cpu_doing_single_step); if (kgdb_info[sstep_cpu].task) kgdb_sstep_pid = kgdb_info[sstep_cpu].task->pid; else kgdb_sstep_pid = 0; } if (arch_kgdb_ops.correct_hw_break) arch_kgdb_ops.correct_hw_break(); if (trace_on) tracing_on(); kgdb_info[cpu].exception_state &= ~(DCPU_WANT_MASTER | DCPU_IS_SLAVE); kgdb_info[cpu].enter_kgdb--; smp_mb__before_atomic(); atomic_dec(&masters_in_kgdb); /* Free kgdb_active */ atomic_set(&kgdb_active, -1); raw_spin_unlock(&dbg_master_lock); dbg_touch_watchdogs(); local_irq_restore(flags); return kgdb_info[cpu].ret_state; }
static int _nfs42_proc_fallocate(struct rpc_message *msg, struct file *filep, struct nfs_lock_context *lock, loff_t offset, loff_t len) { struct inode *inode = file_inode(filep); struct nfs_server *server = NFS_SERVER(inode); struct nfs42_falloc_args args = { .falloc_fh = NFS_FH(inode), .falloc_offset = offset, .falloc_length = len, .falloc_bitmask = server->cache_consistency_bitmask, }; struct nfs42_falloc_res res = { .falloc_server = server, }; int status; msg->rpc_argp = &args; msg->rpc_resp = &res; status = nfs4_set_rw_stateid(&args.falloc_stateid, lock->open_context, lock, FMODE_WRITE); if (status) return status; res.falloc_fattr = nfs_alloc_fattr(); if (!res.falloc_fattr) return -ENOMEM; status = nfs4_call_sync(server->client, server, msg, &args.seq_args, &res.seq_res, 0); if (status == 0) status = nfs_post_op_update_inode(inode, res.falloc_fattr); kfree(res.falloc_fattr); return status; } static int nfs42_proc_fallocate(struct rpc_message *msg, struct file *filep, loff_t offset, loff_t len) { struct nfs_server *server = NFS_SERVER(file_inode(filep)); struct nfs4_exception exception = { }; struct nfs_lock_context *lock; int err; lock = nfs_get_lock_context(nfs_file_open_context(filep)); if (IS_ERR(lock)) return PTR_ERR(lock); exception.inode = file_inode(filep); exception.state = lock->open_context->state; do { err = _nfs42_proc_fallocate(msg, filep, lock, offset, len); if (err == -ENOTSUPP) { err = -EOPNOTSUPP; break; } err = nfs4_handle_exception(server, err, &exception); } while (exception.retry); nfs_put_lock_context(lock); return err; } int nfs42_proc_allocate(struct file *filep, loff_t offset, loff_t len) { struct rpc_message msg = { .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_ALLOCATE], }; struct inode *inode = file_inode(filep); int err; if (!nfs_server_capable(inode, NFS_CAP_ALLOCATE)) return -EOPNOTSUPP; inode_lock(inode); err = nfs42_proc_fallocate(&msg, filep, offset, len); if (err == -EOPNOTSUPP) NFS_SERVER(inode)->caps &= ~NFS_CAP_ALLOCATE; inode_unlock(inode); return err; } int nfs42_proc_deallocate(struct file *filep, loff_t offset, loff_t len) { struct rpc_message msg = { .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_DEALLOCATE], }; struct inode *inode = file_inode(filep); int err; if (!nfs_server_capable(inode, NFS_CAP_DEALLOCATE)) return -EOPNOTSUPP; inode_lock(inode); err = nfs_sync_inode(inode); if (err) goto out_unlock; err = nfs42_proc_fallocate(&msg, filep, offset, len); if (err == 0) truncate_pagecache_range(inode, offset, (offset + len) -1); if (err == -EOPNOTSUPP) NFS_SERVER(inode)->caps &= ~NFS_CAP_DEALLOCATE; out_unlock: inode_unlock(inode); return err; } static ssize_t _nfs42_proc_copy(struct file *src, loff_t pos_src, struct nfs_lock_context *src_lock, struct file *dst, loff_t pos_dst, struct nfs_lock_context *dst_lock, size_t count) { struct nfs42_copy_args args = { .src_fh = NFS_FH(file_inode(src)), .src_pos = pos_src, .dst_fh = NFS_FH(file_inode(dst)), .dst_pos = pos_dst, .count = count, }; struct nfs42_copy_res res; struct rpc_message msg = { .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_COPY], .rpc_argp = &args, .rpc_resp = &res, }; struct inode *dst_inode = file_inode(dst); struct nfs_server *server = NFS_SERVER(dst_inode); int status; status = nfs4_set_rw_stateid(&args.src_stateid, src_lock->open_context, src_lock, FMODE_READ); if (status) return status; status = nfs_filemap_write_and_wait_range(file_inode(src)->i_mapping, pos_src, pos_src + (loff_t)count - 1); if (status) return status; status = nfs4_set_rw_stateid(&args.dst_stateid, dst_lock->open_context, dst_lock, FMODE_WRITE); if (status) return status; status = nfs_sync_inode(dst_inode); if (status) return status; status = nfs4_call_sync(server->client, server, &msg, &args.seq_args, &res.seq_res, 0); if (status == -ENOTSUPP) server->caps &= ~NFS_CAP_COPY; if (status) return status; if (res.write_res.verifier.committed != NFS_FILE_SYNC) { status = nfs_commit_file(dst, &res.write_res.verifier.verifier); if (status) return status; } truncate_pagecache_range(dst_inode, pos_dst, pos_dst + res.write_res.count); return res.write_res.count; } ssize_t nfs42_proc_copy(struct file *src, loff_t pos_src, struct file *dst, loff_t pos_dst, size_t count) { struct nfs_server *server = NFS_SERVER(file_inode(dst)); struct nfs_lock_context *src_lock; struct nfs_lock_context *dst_lock; struct nfs4_exception src_exception = { }; struct nfs4_exception dst_exception = { }; ssize_t err, err2; if (!nfs_server_capable(file_inode(dst), NFS_CAP_COPY)) return -EOPNOTSUPP; src_lock = nfs_get_lock_context(nfs_file_open_context(src)); if (IS_ERR(src_lock)) return PTR_ERR(src_lock); src_exception.inode = file_inode(src); src_exception.state = src_lock->open_context->state; dst_lock = nfs_get_lock_context(nfs_file_open_context(dst)); if (IS_ERR(dst_lock)) { err = PTR_ERR(dst_lock); goto out_put_src_lock; } dst_exception.inode = file_inode(dst); dst_exception.state = dst_lock->open_context->state; do { inode_lock(file_inode(dst)); err = _nfs42_proc_copy(src, pos_src, src_lock, dst, pos_dst, dst_lock, count); inode_unlock(file_inode(dst)); if (err == -ENOTSUPP) { err = -EOPNOTSUPP; break; } err2 = nfs4_handle_exception(server, err, &src_exception); err = nfs4_handle_exception(server, err, &dst_exception); if (!err) err = err2; } while (src_exception.retry || dst_exception.retry); nfs_put_lock_context(dst_lock); out_put_src_lock: nfs_put_lock_context(src_lock); return err; } static loff_t _nfs42_proc_llseek(struct file *filep, struct nfs_lock_context *lock, loff_t offset, int whence) { struct inode *inode = file_inode(filep); struct nfs42_seek_args args = { .sa_fh = NFS_FH(inode), .sa_offset = offset, .sa_what = (whence == SEEK_HOLE) ? NFS4_CONTENT_HOLE : NFS4_CONTENT_DATA, }; struct nfs42_seek_res res; struct rpc_message msg = { .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_SEEK], .rpc_argp = &args, .rpc_resp = &res, }; struct nfs_server *server = NFS_SERVER(inode); int status; if (!nfs_server_capable(inode, NFS_CAP_SEEK)) return -ENOTSUPP; status = nfs4_set_rw_stateid(&args.sa_stateid, lock->open_context, lock, FMODE_READ); if (status) return status; status = nfs_filemap_write_and_wait_range(inode->i_mapping, offset, LLONG_MAX); if (status) return status; status = nfs4_call_sync(server->client, server, &msg, &args.seq_args, &res.seq_res, 0); if (status == -ENOTSUPP) server->caps &= ~NFS_CAP_SEEK; if (status) return status; return vfs_setpos(filep, res.sr_offset, inode->i_sb->s_maxbytes); } loff_t nfs42_proc_llseek(struct file *filep, loff_t offset, int whence) { struct nfs_server *server = NFS_SERVER(file_inode(filep)); struct nfs4_exception exception = { }; struct nfs_lock_context *lock; loff_t err; lock = nfs_get_lock_context(nfs_file_open_context(filep)); if (IS_ERR(lock)) return PTR_ERR(lock); exception.inode = file_inode(filep); exception.state = lock->open_context->state; do { err = _nfs42_proc_llseek(filep, lock, offset, whence); if (err >= 0) break; if (err == -ENOTSUPP) { err = -EOPNOTSUPP; break; } err = nfs4_handle_exception(server, err, &exception); } while (exception.retry); nfs_put_lock_context(lock); return err; } static void nfs42_layoutstat_prepare(struct rpc_task *task, void *calldata) { struct nfs42_layoutstat_data *data = calldata; struct inode *inode = data->inode; struct nfs_server *server = NFS_SERVER(inode); struct pnfs_layout_hdr *lo; spin_lock(&inode->i_lock); lo = NFS_I(inode)->layout; if (!pnfs_layout_is_valid(lo)) { spin_unlock(&inode->i_lock); rpc_exit(task, 0); return; } nfs4_stateid_copy(&data->args.stateid, &lo->plh_stateid); spin_unlock(&inode->i_lock); nfs41_setup_sequence(nfs4_get_session(server), &data->args.seq_args, &data->res.seq_res, task); } static void nfs42_layoutstat_done(struct rpc_task *task, void *calldata) { struct nfs42_layoutstat_data *data = calldata; struct inode *inode = data->inode; struct pnfs_layout_hdr *lo; if (!nfs4_sequence_done(task, &data->res.seq_res)) return; switch (task->tk_status) { case 0: break; case -NFS4ERR_EXPIRED: case -NFS4ERR_ADMIN_REVOKED: case -NFS4ERR_DELEG_REVOKED: case -NFS4ERR_STALE_STATEID: case -NFS4ERR_BAD_STATEID: spin_lock(&inode->i_lock); lo = NFS_I(inode)->layout; if (pnfs_layout_is_valid(lo) && nfs4_stateid_match(&data->args.stateid, &lo->plh_stateid)) { LIST_HEAD(head); /* * Mark the bad layout state as invalid, then retry * with the current stateid. */ pnfs_mark_layout_stateid_invalid(lo, &head); spin_unlock(&inode->i_lock); pnfs_free_lseg_list(&head); } else spin_unlock(&inode->i_lock); break; case -NFS4ERR_OLD_STATEID: spin_lock(&inode->i_lock); lo = NFS_I(inode)->layout; if (pnfs_layout_is_valid(lo) && nfs4_stateid_match_other(&data->args.stateid, &lo->plh_stateid)) { /* Do we need to delay before resending? */ if (!nfs4_stateid_is_newer(&lo->plh_stateid, &data->args.stateid)) rpc_delay(task, HZ); rpc_restart_call_prepare(task); } spin_unlock(&inode->i_lock); break; case -ENOTSUPP: case -EOPNOTSUPP: NFS_SERVER(inode)->caps &= ~NFS_CAP_LAYOUTSTATS; } dprintk("%s server returns %d\n", __func__, task->tk_status); } static void nfs42_layoutstat_release(void *calldata) { struct nfs42_layoutstat_data *data = calldata; struct nfs_server *nfss = NFS_SERVER(data->args.inode); if (nfss->pnfs_curr_ld->cleanup_layoutstats) nfss->pnfs_curr_ld->cleanup_layoutstats(data); pnfs_put_layout_hdr(NFS_I(data->args.inode)->layout); smp_mb__before_atomic(); clear_bit(NFS_INO_LAYOUTSTATS, &NFS_I(data->args.inode)->flags); smp_mb__after_atomic(); nfs_iput_and_deactive(data->inode); kfree(data->args.devinfo); kfree(data); } static const struct rpc_call_ops nfs42_layoutstat_ops = { .rpc_call_prepare = nfs42_layoutstat_prepare, .rpc_call_done = nfs42_layoutstat_done, .rpc_release = nfs42_layoutstat_release, }; int nfs42_proc_layoutstats_generic(struct nfs_server *server, struct nfs42_layoutstat_data *data) { struct rpc_message msg = { .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_LAYOUTSTATS], .rpc_argp = &data->args, .rpc_resp = &data->res, }; struct rpc_task_setup task_setup = { .rpc_client = server->client, .rpc_message = &msg, .callback_ops = &nfs42_layoutstat_ops, .callback_data = data, .flags = RPC_TASK_ASYNC, }; struct rpc_task *task; data->inode = nfs_igrab_and_active(data->args.inode); if (!data->inode) { nfs42_layoutstat_release(data); return -EAGAIN; } nfs4_init_sequence(&data->args.seq_args, &data->res.seq_res, 0); task = rpc_run_task(&task_setup); if (IS_ERR(task)) return PTR_ERR(task); rpc_put_task(task); return 0; } static int _nfs42_proc_clone(struct rpc_message *msg, struct file *src_f, struct file *dst_f, struct nfs_lock_context *src_lock, struct nfs_lock_context *dst_lock, loff_t src_offset, loff_t dst_offset, loff_t count) { struct inode *src_inode = file_inode(src_f); struct inode *dst_inode = file_inode(dst_f); struct nfs_server *server = NFS_SERVER(dst_inode); struct nfs42_clone_args args = { .src_fh = NFS_FH(src_inode), .dst_fh = NFS_FH(dst_inode), .src_offset = src_offset, .dst_offset = dst_offset, .count = count, .dst_bitmask = server->cache_consistency_bitmask, }; struct nfs42_clone_res res = { .server = server, }; int status; msg->rpc_argp = &args; msg->rpc_resp = &res; status = nfs4_set_rw_stateid(&args.src_stateid, src_lock->open_context, src_lock, FMODE_READ); if (status) return status; status = nfs4_set_rw_stateid(&args.dst_stateid, dst_lock->open_context, dst_lock, FMODE_WRITE); if (status) return status; res.dst_fattr = nfs_alloc_fattr(); if (!res.dst_fattr) return -ENOMEM; status = nfs4_call_sync(server->client, server, msg, &args.seq_args, &res.seq_res, 0); if (status == 0) status = nfs_post_op_update_inode(dst_inode, res.dst_fattr); kfree(res.dst_fattr); return status; } int nfs42_proc_clone(struct file *src_f, struct file *dst_f, loff_t src_offset, loff_t dst_offset, loff_t count) { struct rpc_message msg = { .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_CLONE], }; struct inode *inode = file_inode(src_f); struct nfs_server *server = NFS_SERVER(file_inode(src_f)); struct nfs_lock_context *src_lock; struct nfs_lock_context *dst_lock; struct nfs4_exception src_exception = { }; struct nfs4_exception dst_exception = { }; int err, err2; if (!nfs_server_capable(inode, NFS_CAP_CLONE)) return -EOPNOTSUPP; src_lock = nfs_get_lock_context(nfs_file_open_context(src_f)); if (IS_ERR(src_lock)) return PTR_ERR(src_lock); src_exception.inode = file_inode(src_f); src_exception.state = src_lock->open_context->state; dst_lock = nfs_get_lock_context(nfs_file_open_context(dst_f)); if (IS_ERR(dst_lock)) { err = PTR_ERR(dst_lock); goto out_put_src_lock; } dst_exception.inode = file_inode(dst_f); dst_exception.state = dst_lock->open_context->state; do { err = _nfs42_proc_clone(&msg, src_f, dst_f, src_lock, dst_lock, src_offset, dst_offset, count); if (err == -ENOTSUPP || err == -EOPNOTSUPP) { NFS_SERVER(inode)->caps &= ~NFS_CAP_CLONE; err = -EOPNOTSUPP; break; } err2 = nfs4_handle_exception(server, err, &src_exception); err = nfs4_handle_exception(server, err, &dst_exception); if (!err) err = err2; } while (src_exception.retry || dst_exception.retry); nfs_put_lock_context(dst_lock); out_put_src_lock: nfs_put_lock_context(src_lock); return err; }
int watchdog_nmi_enable(unsigned int cpu) { struct perf_event_attr *wd_attr; struct perf_event *event = per_cpu(watchdog_ev, cpu); /* nothing to do if the hard lockup detector is disabled */ if (!(watchdog_enabled & NMI_WATCHDOG_ENABLED)) goto out; /* is it already setup and enabled? */ if (event && event->state > PERF_EVENT_STATE_OFF) goto out; /* it is setup but not enabled */ if (event != NULL) goto out_enable; wd_attr = &wd_hw_attr; wd_attr->sample_period = hw_nmi_get_sample_period(watchdog_thresh); /* Try to register using hardware perf events */ event = perf_event_create_kernel_counter(wd_attr, cpu, NULL, watchdog_overflow_callback, NULL); /* save cpu0 error for future comparision */ if (cpu == 0 && IS_ERR(event)) cpu0_err = PTR_ERR(event); if (!IS_ERR(event)) { /* only print for cpu0 or different than cpu0 */ if (cpu == 0 || cpu0_err) pr_info("enabled on all CPUs, permanently consumes one hw-PMU counter.\n"); goto out_save; } /* * Disable the hard lockup detector if _any_ CPU fails to set up * set up the hardware perf event. The watchdog() function checks * the NMI_WATCHDOG_ENABLED bit periodically. * * The barriers are for syncing up watchdog_enabled across all the * cpus, as clear_bit() does not use barriers. */ smp_mb__before_atomic(); clear_bit(NMI_WATCHDOG_ENABLED_BIT, &watchdog_enabled); smp_mb__after_atomic(); /* skip displaying the same error again */ if (cpu > 0 && (PTR_ERR(event) == cpu0_err)) return PTR_ERR(event); /* vary the KERN level based on the returned errno */ if (PTR_ERR(event) == -EOPNOTSUPP) pr_info("disabled (cpu%i): not supported (no LAPIC?)\n", cpu); else if (PTR_ERR(event) == -ENOENT) pr_warn("disabled (cpu%i): hardware events not enabled\n", cpu); else pr_err("disabled (cpu%i): unable to create perf event: %ld\n", cpu, PTR_ERR(event)); pr_info("Shutting down hard lockup detector on all cpus\n"); return PTR_ERR(event); /* success path */ out_save: per_cpu(watchdog_ev, cpu) = event; out_enable: perf_event_enable(per_cpu(watchdog_ev, cpu)); out: return 0; }