void synchronise_count_slave(int cpu)
{
int i;
/*
* Not every cpu is online at the time this gets called,
* so we first wait for the master to say everyone is ready
*/
for (i = 0; i < NR_LOOPS; i++) {
atomic_inc_unchecked(&count_count_start);
while (atomic_read_unchecked(&count_count_start) != 2)
mb();
/*
* Everyone initialises count in the last loop:
*/
if (i == NR_LOOPS-1)
write_c0_count(initcount);
atomic_inc_unchecked(&count_count_stop);
while (atomic_read_unchecked(&count_count_stop) != 2)
mb();
}
/* Arrange for an interrupt in a short while */
write_c0_compare(read_c0_count() + COUNTON);
}
static int giu_get_irq(unsigned int irq)
{
u16 pendl, pendh, maskl, maskh;
int i;
pendl = giu_read(GIUINTSTATL);
pendh = giu_read(GIUINTSTATH);
maskl = giu_read(GIUINTENL);
maskh = giu_read(GIUINTENH);
maskl &= pendl;
maskh &= pendh;
if (maskl) {
for (i = 0; i < 16; i++) {
if (maskl & (1 << i))
return GIU_IRQ(i);
}
} else if (maskh) {
for (i = 0; i < 16; i++) {
if (maskh & (1 << i))
return GIU_IRQ(i + GIUINT_HIGH_OFFSET);
}
}
printk(KERN_ERR "spurious GIU interrupt: %04x(%04x),%04x(%04x)\n",
maskl, pendl, maskh, pendh);
atomic_inc_unchecked(&irq_err_count);
return -EINVAL;
}
/**
* uio_event_notify - trigger an interrupt event
* @info: UIO device capabilities
*/
void uio_event_notify(struct uio_info *info)
{
struct uio_device *idev = info->uio_dev;
atomic_inc_unchecked(&idev->event);
wake_up_interruptible(&idev->wait);
kill_fasync(&idev->async_queue, SIGIO, POLL_IN);
}
static void switch_worker(struct work_struct *work)
{
if (oprofile_ops.switch_events())
return;
atomic_inc_unchecked(&oprofile_stats.multiplex_counter);
start_switch_worker();
}
/*
* Free an element to the rcu_tortures pool.
*/
static void
rcu_torture_free(struct rcu_torture *p)
{
atomic_inc_unchecked(&n_rcu_torture_free);
spin_lock_bh(&rcu_torture_lock);
list_add_tail(&p->rtort_free, &rcu_torture_freelist);
spin_unlock_bh(&rcu_torture_lock);
}
/*
* Allocate an element from the rcu_tortures pool.
*/
static struct rcu_torture *
rcu_torture_alloc(void)
{
struct list_head *p;
spin_lock_bh(&rcu_torture_lock);
if (list_empty(&rcu_torture_freelist)) {
atomic_inc_unchecked(&n_rcu_torture_alloc_fail);
spin_unlock_bh(&rcu_torture_lock);
return NULL;
}
atomic_inc_unchecked(&n_rcu_torture_alloc);
p = rcu_torture_freelist.next;
list_del_init(p);
spin_unlock_bh(&rcu_torture_lock);
return container_of(p, struct rcu_torture, rtort_free);
}
int atm_charge(struct atm_vcc *vcc, int truesize)
{
atm_force_charge(vcc, truesize);
if (atomic_read(&sk_atm(vcc)->sk_rmem_alloc) <= sk_atm(vcc)->sk_rcvbuf)
return 1;
atm_return(vcc, truesize);
atomic_inc_unchecked(&vcc->stats->rx_drop);
return 0;
}
static irqreturn_t sht15_interrupt_fired(int irq, void *d)
{
struct sht15_data *data = d;
/* First disable the interrupt */
disable_irq_nosync(irq);
atomic_inc_unchecked(&data->interrupt_handled);
/* Then schedule a reading work struct */
if (data->flag != SHT15_READING_NOTHING)
schedule_work(&data->read_work);
return IRQ_HANDLED;
}
irqreturn_t mga_driver_irq_handler(DRM_IRQ_ARGS)
{
struct drm_device *dev = (struct drm_device *) arg;
drm_mga_private_t *dev_priv = (drm_mga_private_t *) dev->dev_private;
int status;
int handled = 0;
status = MGA_READ(MGA_STATUS);
/* VBLANK interrupt */
if (status & MGA_VLINEPEN) {
MGA_WRITE(MGA_ICLEAR, MGA_VLINEICLR);
atomic_inc_unchecked(&dev_priv->vbl_received);
drm_handle_vblank(dev, 0);
handled = 1;
}
/* SOFTRAP interrupt */
if (status & MGA_SOFTRAPEN) {
const u32 prim_start = MGA_READ(MGA_PRIMADDRESS);
const u32 prim_end = MGA_READ(MGA_PRIMEND);
MGA_WRITE(MGA_ICLEAR, MGA_SOFTRAPICLR);
/* In addition to clearing the interrupt-pending bit, we
* have to write to MGA_PRIMEND to re-start the DMA operation.
*/
if ((prim_start & ~0x03) != (prim_end & ~0x03))
MGA_WRITE(MGA_PRIMEND, prim_end);
atomic_inc_unchecked(&dev_priv->last_fence_retired);
DRM_WAKEUP(&dev_priv->fence_queue);
handled = 1;
}
if (handled)
return IRQ_HANDLED;
return IRQ_NONE;
}
int raw_rcv(struct sock *sk, struct sk_buff *skb)
{
if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb)) {
atomic_inc_unchecked(&sk->sk_drops);
kfree_skb(skb);
return NET_RX_DROP;
}
nf_reset(skb);
skb_push(skb, skb->data - skb_network_header(skb));
raw_rcv_skb(sk, skb);
return 0;
}
void sysfs_notify_dirent(struct sysfs_dirent *sd)
{
struct sysfs_open_dirent *od;
unsigned long flags;
spin_lock_irqsave(&sysfs_open_dirent_lock, flags);
od = sd->s_attr.open;
if (od) {
atomic_inc_unchecked(&od->event);
wake_up_interruptible(&od->poll);
}
spin_unlock_irqrestore(&sysfs_open_dirent_lock, flags);
}
static irqreturn_t lis302dl_interrupt(int irq, void *dummy)
{
if (!test_bit(0, &lis3_dev.misc_opened))
goto out;
/*
* Be careful: on some HP laptops the bios force DD when on battery and
* the lid is closed. This leads to interrupts as soon as a little move
* is done.
*/
atomic_inc_unchecked(&lis3_dev.count);
wake_up_interruptible(&lis3_dev.misc_wait);
kill_fasync(&lis3_dev.async_queue, SIGIO, POLL_IN);
out:
if (atomic_read(&lis3_dev.wake_thread))
return IRQ_WAKE_THREAD;
return IRQ_HANDLED;
}
struct sk_buff *atm_alloc_charge(struct atm_vcc *vcc, int pdu_size,
gfp_t gfp_flags)
{
struct sock *sk = sk_atm(vcc);
int guess = SKB_TRUESIZE(pdu_size);
atm_force_charge(vcc, guess);
if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf) {
struct sk_buff *skb = alloc_skb(pdu_size, gfp_flags);
if (skb) {
atomic_add(skb->truesize-guess,
&sk->sk_rmem_alloc);
return skb;
}
}
atm_return(vcc, guess);
atomic_inc_unchecked(&vcc->stats->rx_drop);
return NULL;
}
void gt641xx_irq_dispatch(void)
{
u32 cause, mask;
int i;
cause = GT_READ(GT_INTRCAUSE_OFS);
mask = GT_READ(GT_INTRMASK_OFS);
cause &= mask;
/*
* bit0 : logical or of all the interrupt bits.
* bit30: logical or of bits[29:26,20:1].
* bit31: logical or of bits[25:1].
*/
for (i = 1; i < 30; i++) {
if (cause & (1U << i)) {
do_IRQ(GT641XX_IRQ_BASE + i);
return;
}
}
atomic_inc_unchecked(&irq_err_count);
}
int
ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
struct ip_vs_protocol *pp, int offset)
{
struct rt6_info *rt; /* Route to the other host */
int mtu;
int rc;
int local;
EnterFunction(10);
/* The ICMP packet for VS/TUN, VS/DR and LOCALNODE will be
forwarded directly here, because there is no need to
translate address/port back */
if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ) {
if (cp->packet_xmit)
rc = cp->packet_xmit(skb, cp, pp);
else
rc = NF_ACCEPT;
/* do not touch skb anymore */
atomic_inc_unchecked(&cp->in_pkts);
goto out;
}
/*
* mangle and send the packet here (only for VS/NAT)
*/
if (!(rt = __ip_vs_get_out_rt_v6(skb, cp->dest, &cp->daddr.in6, NULL,
0, 1|2|4)))
goto tx_error_icmp;
local = __ip_vs_is_local_route6(rt);
/*
* Avoid duplicate tuple in reply direction for NAT traffic
* to local address when connection is sync-ed
*/
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
if (cp->flags & IP_VS_CONN_F_SYNC && local) {
enum ip_conntrack_info ctinfo;
struct nf_conn *ct = ct = nf_ct_get(skb, &ctinfo);
if (ct && !nf_ct_is_untracked(ct)) {
IP_VS_DBG(10, "%s(): "
"stopping DNAT to local address %pI6\n",
__func__, &cp->daddr.in6);
goto tx_error_put;
}
}
#endif
/* From world but DNAT to loopback address? */
if (local && skb->dev && !(skb->dev->flags & IFF_LOOPBACK) &&
ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_LOOPBACK) {
IP_VS_DBG(1, "%s(): "
"stopping DNAT to loopback %pI6\n",
__func__, &cp->daddr.in6);
goto tx_error_put;
}
/* MTU checking */
mtu = dst_mtu(&rt->dst);
if (skb->len > mtu && !skb_is_gso(skb)) {
if (!skb->dev) {
struct net *net = dev_net(skb_dst(skb)->dev);
skb->dev = net->loopback_dev;
}
icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
IP_VS_DBG_RL("%s(): frag needed\n", __func__);
goto tx_error_put;
}
/* copy-on-write the packet before mangling it */
if (!skb_make_writable(skb, offset))
goto tx_error_put;
if (skb_cow(skb, rt->dst.dev->hard_header_len))
goto tx_error_put;
ip_vs_nat_icmp_v6(skb, pp, cp, 0);
if (!local || !skb->dev) {
/* drop the old route when skb is not shared */
skb_dst_drop(skb);
skb_dst_set(skb, &rt->dst);
} else {
/* destined to loopback, do we need to change route? */
dst_release(&rt->dst);
}
/* Another hack: avoid icmp_send in ip_fragment */
skb->local_df = 1;
IP_VS_XMIT_NAT(NFPROTO_IPV6, skb, cp, local);
rc = NF_STOLEN;
goto out;
tx_error_icmp:
dst_link_failure(skb);
tx_error:
dev_kfree_skb(skb);
rc = NF_STOLEN;
out:
LeaveFunction(10);
return rc;
tx_error_put:
dst_release(&rt->dst);
goto tx_error;
}
/*
* ICMP packet transmitter
* called by the ip_vs_in_icmp
*/
int
ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
struct ip_vs_protocol *pp, int offset)
{
struct rtable *rt; /* Route to the other host */
int mtu;
int rc;
int local;
EnterFunction(10);
/* The ICMP packet for VS/TUN, VS/DR and LOCALNODE will be
forwarded directly here, because there is no need to
translate address/port back */
if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ) {
if (cp->packet_xmit)
rc = cp->packet_xmit(skb, cp, pp);
else
rc = NF_ACCEPT;
/* do not touch skb anymore */
atomic_inc_unchecked(&cp->in_pkts);
goto out;
}
/*
* mangle and send the packet here (only for VS/NAT)
*/
if (!(rt = __ip_vs_get_out_rt(skb, cp->dest, cp->daddr.ip,
RT_TOS(ip_hdr(skb)->tos),
IP_VS_RT_MODE_LOCAL |
IP_VS_RT_MODE_NON_LOCAL |
IP_VS_RT_MODE_RDR)))
goto tx_error_icmp;
local = rt->rt_flags & RTCF_LOCAL;
/*
* Avoid duplicate tuple in reply direction for NAT traffic
* to local address when connection is sync-ed
*/
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
if (cp->flags & IP_VS_CONN_F_SYNC && local) {
enum ip_conntrack_info ctinfo;
struct nf_conn *ct = ct = nf_ct_get(skb, &ctinfo);
if (ct && !nf_ct_is_untracked(ct)) {
IP_VS_DBG(10, "%s(): "
"stopping DNAT to local address %pI4\n",
__func__, &cp->daddr.ip);
goto tx_error_put;
}
}
#endif
/* From world but DNAT to loopback address? */
if (local && ipv4_is_loopback(rt->rt_dst) &&
rt_is_input_route(skb_rtable(skb))) {
IP_VS_DBG(1, "%s(): "
"stopping DNAT to loopback %pI4\n",
__func__, &cp->daddr.ip);
goto tx_error_put;
}
/* MTU checking */
mtu = dst_mtu(&rt->dst);
if ((skb->len > mtu) && (ip_hdr(skb)->frag_off & htons(IP_DF)) &&
!skb_is_gso(skb)) {
icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED, htonl(mtu));
IP_VS_DBG_RL("%s(): frag needed\n", __func__);
goto tx_error_put;
}
/* copy-on-write the packet before mangling it */
if (!skb_make_writable(skb, offset))
goto tx_error_put;
if (skb_cow(skb, rt->dst.dev->hard_header_len))
goto tx_error_put;
ip_vs_nat_icmp(skb, pp, cp, 0);
if (!local) {
/* drop the old route when skb is not shared */
skb_dst_drop(skb);
skb_dst_set(skb, &rt->dst);
} else {
ip_rt_put(rt);
/*
* Some IPv4 replies get local address from routes,
* not from iph, so while we DNAT after routing
* we need this second input/output route.
*/
if (!__ip_vs_reroute_locally(skb))
goto tx_error;
}
/* Another hack: avoid icmp_send in ip_fragment */
skb->local_df = 1;
IP_VS_XMIT_NAT(NFPROTO_IPV4, skb, cp, local);
rc = NF_STOLEN;
goto out;
tx_error_icmp:
dst_link_failure(skb);
tx_error:
dev_kfree_skb(skb);
rc = NF_STOLEN;
out:
LeaveFunction(10);
return rc;
tx_error_put:
ip_rt_put(rt);
goto tx_error;
}
static void poll_rx(struct atm_dev *dev,int mbx)
{
struct zatm_dev *zatm_dev;
unsigned long pos;
u32 x;
int error;
EVENT("poll_rx\n",0,0);
zatm_dev = ZATM_DEV(dev);
pos = (zatm_dev->mbx_start[mbx] & ~0xffffUL) | zin(MTA(mbx));
while (x = zin(MWA(mbx)), (pos & 0xffff) != x) {
u32 *here;
struct sk_buff *skb;
struct atm_vcc *vcc;
int cells,size,chan;
EVENT("MBX: host 0x%lx, nic 0x%x\n",pos,x);
here = (u32 *) pos;
if (((pos += 16) & 0xffff) == zatm_dev->mbx_end[mbx])
pos = zatm_dev->mbx_start[mbx];
cells = here[0] & uPD98401_AAL5_SIZE;
#if 0
printk("RX IND: 0x%x, 0x%x, 0x%x, 0x%x\n",here[0],here[1],here[2],here[3]);
{
unsigned long *x;
printk("POOL: 0x%08x, 0x%08x\n",zpeekl(zatm_dev,
zatm_dev->pool_base),
zpeekl(zatm_dev,zatm_dev->pool_base+1));
x = (unsigned long *) here[2];
printk("[0..3] = 0x%08lx, 0x%08lx, 0x%08lx, 0x%08lx\n",
x[0],x[1],x[2],x[3]);
}
#endif
error = 0;
if (here[3] & uPD98401_AAL5_ERR) {
error = (here[3] & uPD98401_AAL5_ES) >>
uPD98401_AAL5_ES_SHIFT;
if (error == uPD98401_AAL5_ES_DEACT ||
error == uPD98401_AAL5_ES_FREE) continue;
}
EVENT("error code 0x%x/0x%x\n",(here[3] & uPD98401_AAL5_ES) >>
uPD98401_AAL5_ES_SHIFT,error);
skb = ((struct rx_buffer_head *) bus_to_virt(here[2]))->skb;
__net_timestamp(skb);
#if 0
printk("[-3..0] 0x%08lx 0x%08lx 0x%08lx 0x%08lx\n",((unsigned *) skb->data)[-3],
((unsigned *) skb->data)[-2],((unsigned *) skb->data)[-1],
((unsigned *) skb->data)[0]);
#endif
EVENT("skb 0x%lx, here 0x%lx\n",(unsigned long) skb,
(unsigned long) here);
#if 0
printk("dummy: 0x%08lx, 0x%08lx\n",dummy[0],dummy[1]);
#endif
size = error ? 0 : ntohs(((__be16 *) skb->data)[cells*
ATM_CELL_PAYLOAD/sizeof(u16)-3]);
EVENT("got skb 0x%lx, size %d\n",(unsigned long) skb,size);
chan = (here[3] & uPD98401_AAL5_CHAN) >>
uPD98401_AAL5_CHAN_SHIFT;
if (chan < zatm_dev->chans && zatm_dev->rx_map[chan]) {
int pos;
vcc = zatm_dev->rx_map[chan];
pos = ZATM_VCC(vcc)->pool;
if (skb == zatm_dev->last_free[pos])
zatm_dev->last_free[pos] = NULL;
skb_unlink(skb, zatm_dev->pool + pos);
}
else {
printk(KERN_ERR DEV_LABEL "(itf %d): RX indication "
"for non-existing channel\n",dev->number);
size = 0;
vcc = NULL;
event_dump();
}
if (error) {
static unsigned long silence = 0;
static int last_error = 0;
if (error != last_error ||
time_after(jiffies, silence) || silence == 0){
printk(KERN_WARNING DEV_LABEL "(itf %d): "
"chan %d error %s\n",dev->number,chan,
err_txt[error]);
last_error = error;
silence = (jiffies+2*HZ)|1;
}
size = 0;
}
if (size && (size > cells*ATM_CELL_PAYLOAD-ATM_AAL5_TRAILER ||
size <= (cells-1)*ATM_CELL_PAYLOAD-ATM_AAL5_TRAILER)) {
printk(KERN_ERR DEV_LABEL "(itf %d): size %d with %d "
"cells\n",dev->number,size,cells);
size = 0;
event_dump();
}
if (size > ATM_MAX_AAL5_PDU) {
printk(KERN_ERR DEV_LABEL "(itf %d): size too big "
"(%d)\n",dev->number,size);
size = 0;
event_dump();
}
if (!size) {
dev_kfree_skb_irq(skb);
if (vcc) atomic_inc_unchecked(&vcc->stats->rx_err);
continue;
}
if (!atm_charge(vcc,skb->truesize)) {
dev_kfree_skb_irq(skb);
continue;
}
skb->len = size;
ATM_SKB(skb)->vcc = vcc;
vcc->push(vcc,skb);
atomic_inc_unchecked(&vcc->stats->rx);
}
void synchronise_count_master(int cpu)
{
int i;
unsigned long flags;
printk(KERN_INFO "Synchronize counters for CPU %u: ", cpu);
local_irq_save(flags);
/*
* We loop a few times to get a primed instruction cache,
* then the last pass is more or less synchronised and
* the master and slaves each set their cycle counters to a known
* value all at once. This reduces the chance of having random offsets
* between the processors, and guarantees that the maximum
* delay between the cycle counters is never bigger than
* the latency of information-passing (cachelines) between
* two CPUs.
*/
for (i = 0; i < NR_LOOPS; i++) {
/* slaves loop on '!= 2' */
while (atomic_read_unchecked(&count_count_start) != 1)
mb();
atomic_set_unchecked(&count_count_stop, 0);
smp_wmb();
/* Let the slave writes its count register */
atomic_inc_unchecked(&count_count_start);
/* Count will be initialised to current timer */
if (i == 1)
initcount = read_c0_count();
/*
* Everyone initialises count in the last loop:
*/
if (i == NR_LOOPS-1)
write_c0_count(initcount);
/*
* Wait for slave to leave the synchronization point:
*/
while (atomic_read_unchecked(&count_count_stop) != 1)
mb();
atomic_set_unchecked(&count_count_start, 0);
smp_wmb();
atomic_inc_unchecked(&count_count_stop);
}
/* Arrange for an interrupt in a short while */
write_c0_compare(read_c0_count() + COUNTON);
local_irq_restore(flags);
/*
* i386 code reported the skew here, but the
* count registers were almost certainly out of sync
* so no point in alarming people
*/
printk("done.\n");
}
int
ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
struct ip_vs_protocol *pp, int offset, unsigned int hooknum,
struct ip_vs_iphdr *ipvsh)
{
struct rt6_info *rt; /* Route to the other host */
int rc;
int local;
int rt_mode;
EnterFunction(10);
/* The ICMP packet for VS/TUN, VS/DR and LOCALNODE will be
forwarded directly here, because there is no need to
translate address/port back */
if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ) {
if (cp->packet_xmit)
rc = cp->packet_xmit(skb, cp, pp, ipvsh);
else
rc = NF_ACCEPT;
/* do not touch skb anymore */
atomic_inc_unchecked(&cp->in_pkts);
goto out;
}
/*
* mangle and send the packet here (only for VS/NAT)
*/
/* LOCALNODE from FORWARD hook is not supported */
rt_mode = (hooknum != NF_INET_FORWARD) ?
IP_VS_RT_MODE_LOCAL | IP_VS_RT_MODE_NON_LOCAL |
IP_VS_RT_MODE_RDR : IP_VS_RT_MODE_NON_LOCAL;
rcu_read_lock();
local = __ip_vs_get_out_rt_v6(skb, cp->dest, &cp->daddr.in6, NULL,
ipvsh, 0, rt_mode);
if (local < 0)
goto tx_error;
rt = (struct rt6_info *) skb_dst(skb);
/*
* Avoid duplicate tuple in reply direction for NAT traffic
* to local address when connection is sync-ed
*/
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
if (cp->flags & IP_VS_CONN_F_SYNC && local) {
enum ip_conntrack_info ctinfo;
struct nf_conn *ct = nf_ct_get(skb, &ctinfo);
if (ct && !nf_ct_is_untracked(ct)) {
IP_VS_DBG(10, "%s(): "
"stopping DNAT to local address %pI6\n",
__func__, &cp->daddr.in6);
goto tx_error;
}
}
#endif
/* From world but DNAT to loopback address? */
if (local && skb->dev && !(skb->dev->flags & IFF_LOOPBACK) &&
ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_LOOPBACK) {
IP_VS_DBG(1, "%s(): "
"stopping DNAT to loopback %pI6\n",
__func__, &cp->daddr.in6);
goto tx_error;
}
/* copy-on-write the packet before mangling it */
if (!skb_make_writable(skb, offset))
goto tx_error;
if (skb_cow(skb, rt->dst.dev->hard_header_len))
goto tx_error;
ip_vs_nat_icmp_v6(skb, pp, cp, 0);
/* Another hack: avoid icmp_send in ip_fragment */
skb->local_df = 1;
rc = ip_vs_nat_send_or_cont(NFPROTO_IPV6, skb, cp, local);
rcu_read_unlock();
goto out;
tx_error:
kfree_skb(skb);
rcu_read_unlock();
rc = NF_STOLEN;
out:
LeaveFunction(10);
return rc;
}
/*
* Use RDMA_READ to read data from the advertised client buffer into the
* XDR stream starting at rq_arg.head[0].iov_base.
* Each chunk in the array
* contains the following fields:
* discrim - '1', This isn't used for data placement
* position - The xdr stream offset (the same for every chunk)
* handle - RMR for client memory region
* length - data transfer length
* offset - 64 bit tagged offset in remote memory region
*
* On our side, we need to read into a pagelist. The first page immediately
* follows the RPC header.
*
* This function returns:
* 0 - No error and no read-list found.
*
* 1 - Successful read-list processing. The data is not yet in
* the pagelist and therefore the RPC request must be deferred. The
* I/O completion will enqueue the transport again and
* svc_rdma_recvfrom will complete the request.
*
* <0 - Error processing/posting read-list.
*
* NOTE: The ctxt must not be touched after the last WR has been posted
* because the I/O completion processing may occur on another
* processor and free / modify the context. Ne touche pas!
*/
static int rdma_read_xdr(struct svcxprt_rdma *xprt,
struct rpcrdma_msg *rmsgp,
struct svc_rqst *rqstp,
struct svc_rdma_op_ctxt *hdr_ctxt)
{
struct ib_send_wr read_wr;
struct ib_send_wr inv_wr;
int err = 0;
int ch_no;
int ch_count;
int byte_count;
int sge_count;
u64 sgl_offset;
struct rpcrdma_read_chunk *ch;
struct svc_rdma_op_ctxt *ctxt = NULL;
struct svc_rdma_req_map *rpl_map;
struct svc_rdma_req_map *chl_map;
/* If no read list is present, return 0 */
ch = svc_rdma_get_read_chunk(rmsgp);
if (!ch)
return 0;
svc_rdma_rcl_chunk_counts(ch, &ch_count, &byte_count);
if (ch_count > RPCSVC_MAXPAGES)
return -EINVAL;
/* Allocate temporary reply and chunk maps */
rpl_map = svc_rdma_get_req_map();
chl_map = svc_rdma_get_req_map();
if (!xprt->sc_frmr_pg_list_len)
sge_count = map_read_chunks(xprt, rqstp, hdr_ctxt, rmsgp,
rpl_map, chl_map, ch_count,
byte_count);
else
sge_count = fast_reg_read_chunks(xprt, rqstp, hdr_ctxt, rmsgp,
rpl_map, chl_map, ch_count,
byte_count);
if (sge_count < 0) {
err = -EIO;
goto out;
}
sgl_offset = 0;
ch_no = 0;
for (ch = (struct rpcrdma_read_chunk *)&rmsgp->rm_body.rm_chunks[0];
ch->rc_discrim != 0; ch++, ch_no++) {
next_sge:
ctxt = svc_rdma_get_context(xprt);
ctxt->direction = DMA_FROM_DEVICE;
ctxt->frmr = hdr_ctxt->frmr;
ctxt->read_hdr = NULL;
clear_bit(RDMACTXT_F_LAST_CTXT, &ctxt->flags);
clear_bit(RDMACTXT_F_FAST_UNREG, &ctxt->flags);
/* Prepare READ WR */
memset(&read_wr, 0, sizeof read_wr);
read_wr.wr_id = (unsigned long)ctxt;
read_wr.opcode = IB_WR_RDMA_READ;
ctxt->wr_op = read_wr.opcode;
read_wr.send_flags = IB_SEND_SIGNALED;
read_wr.wr.rdma.rkey = ch->rc_target.rs_handle;
read_wr.wr.rdma.remote_addr =
get_unaligned(&(ch->rc_target.rs_offset)) +
sgl_offset;
read_wr.sg_list = ctxt->sge;
read_wr.num_sge =
rdma_read_max_sge(xprt, chl_map->ch[ch_no].count);
err = rdma_set_ctxt_sge(xprt, ctxt, hdr_ctxt->frmr,
&rpl_map->sge[chl_map->ch[ch_no].start],
&sgl_offset,
read_wr.num_sge);
if (err) {
svc_rdma_unmap_dma(ctxt);
svc_rdma_put_context(ctxt, 0);
goto out;
}
if (((ch+1)->rc_discrim == 0) &&
(read_wr.num_sge == chl_map->ch[ch_no].count)) {
/*
* Mark the last RDMA_READ with a bit to
* indicate all RPC data has been fetched from
* the client and the RPC needs to be enqueued.
*/
set_bit(RDMACTXT_F_LAST_CTXT, &ctxt->flags);
if (hdr_ctxt->frmr) {
set_bit(RDMACTXT_F_FAST_UNREG, &ctxt->flags);
/*
* Invalidate the local MR used to map the data
* sink.
*/
if (xprt->sc_dev_caps &
SVCRDMA_DEVCAP_READ_W_INV) {
read_wr.opcode =
IB_WR_RDMA_READ_WITH_INV;
ctxt->wr_op = read_wr.opcode;
read_wr.ex.invalidate_rkey =
ctxt->frmr->mr->lkey;
} else {
/* Prepare INVALIDATE WR */
memset(&inv_wr, 0, sizeof inv_wr);
inv_wr.opcode = IB_WR_LOCAL_INV;
inv_wr.send_flags = IB_SEND_SIGNALED;
inv_wr.ex.invalidate_rkey =
hdr_ctxt->frmr->mr->lkey;
read_wr.next = &inv_wr;
}
}
ctxt->read_hdr = hdr_ctxt;
}
/* Post the read */
err = svc_rdma_send(xprt, &read_wr);
if (err) {
printk(KERN_ERR "svcrdma: Error %d posting RDMA_READ\n",
err);
set_bit(XPT_CLOSE, &xprt->sc_xprt.xpt_flags);
svc_rdma_unmap_dma(ctxt);
svc_rdma_put_context(ctxt, 0);
goto out;
}
atomic_inc_unchecked(&rdma_stat_read);
if (read_wr.num_sge < chl_map->ch[ch_no].count) {
chl_map->ch[ch_no].count -= read_wr.num_sge;
chl_map->ch[ch_no].start += read_wr.num_sge;
goto next_sge;
}
sgl_offset = 0;
err = 1;
}
out:
svc_rdma_put_req_map(rpl_map);
svc_rdma_put_req_map(chl_map);
/* Detach arg pages. svc_recv will replenish them */
for (ch_no = 0; &rqstp->rq_pages[ch_no] < rqstp->rq_respages; ch_no++)
rqstp->rq_pages[ch_no] = NULL;
/*
* Detach res pages. svc_release must see a resused count of
* zero or it will attempt to put them.
*/
while (rqstp->rq_resused)
rqstp->rq_respages[--rqstp->rq_resused] = NULL;
return err;
}
