/*
* Discard a packet we've used up and advance the Rx window by one.
*/
static void rxrpc_rotate_rx_window(struct rxrpc_call *call)
{
struct rxrpc_skb_priv *sp;
struct sk_buff *skb;
rxrpc_serial_t serial;
rxrpc_seq_t hard_ack, top;
u8 flags;
int ix;
_enter("%d", call->debug_id);
hard_ack = call->rx_hard_ack;
top = smp_load_acquire(&call->rx_top);
ASSERT(before(hard_ack, top));
hard_ack++;
ix = hard_ack & RXRPC_RXTX_BUFF_MASK;
skb = call->rxtx_buffer[ix];
rxrpc_see_skb(skb, rxrpc_skb_rx_rotated);
sp = rxrpc_skb(skb);
flags = sp->hdr.flags;
serial = sp->hdr.serial;
if (call->rxtx_annotations[ix] & RXRPC_RX_ANNO_JUMBO)
serial += (call->rxtx_annotations[ix] & RXRPC_RX_ANNO_JUMBO) - 1;
call->rxtx_buffer[ix] = NULL;
call->rxtx_annotations[ix] = 0;
/* Barrier against rxrpc_input_data(). */
smp_store_release(&call->rx_hard_ack, hard_ack);
rxrpc_free_skb(skb, rxrpc_skb_rx_freed);
_debug("%u,%u,%02x", hard_ack, top, flags);
trace_rxrpc_receive(call, rxrpc_receive_rotate, serial, hard_ack);
if (flags & RXRPC_LAST_PACKET) {
rxrpc_end_rx_phase(call, serial);
} else {
/* Check to see if there's an ACK that needs sending. */
if (after_eq(hard_ack, call->ackr_consumed + 2) ||
after_eq(top, call->ackr_seen + 2) ||
(hard_ack == top && after(hard_ack, call->ackr_consumed)))
rxrpc_propose_ACK(call, RXRPC_ACK_DELAY, 0, serial,
true, false,
rxrpc_propose_ack_rotate_rx);
if (call->ackr_reason)
rxrpc_send_call_packet(call, RXRPC_PACKET_TYPE_ACK);
}
}
/*
* queue a packet for recvmsg to pass to userspace
* - the caller must hold a lock on call->lock
* - must not be called with interrupts disabled (sk_filter() disables BH's)
* - eats the packet whether successful or not
* - there must be just one reference to the packet, which the caller passes to
* this function
*/
int rxrpc_queue_rcv_skb(struct rxrpc_call *call, struct sk_buff *skb,
bool force, bool terminal)
{
struct rxrpc_skb_priv *sp;
struct rxrpc_sock *rx = call->socket;
struct sock *sk;
int ret;
_enter(",,%d,%d", force, terminal);
ASSERT(!irqs_disabled());
sp = rxrpc_skb(skb);
ASSERTCMP(sp->call, ==, call);
/* if we've already posted the terminal message for a call, then we
* don't post any more */
if (test_bit(RXRPC_CALL_TERMINAL_MSG, &call->flags)) {
_debug("already terminated");
ASSERTCMP(call->state, >=, RXRPC_CALL_COMPLETE);
rxrpc_free_skb(skb);
return 0;
}
int rxrpc_queue_rcv_skb(struct rxrpc_call *call, struct sk_buff *skb,
bool force, bool terminal)
{
struct rxrpc_skb_priv *sp;
struct rxrpc_sock *rx = call->socket;
struct sock *sk;
int skb_len, ret;
_enter(",,%d,%d", force, terminal);
ASSERT(!irqs_disabled());
sp = rxrpc_skb(skb);
ASSERTCMP(sp->call, ==, call);
if (test_bit(RXRPC_CALL_TERMINAL_MSG, &call->flags)) {
_debug("already terminated");
ASSERTCMP(call->state, >=, RXRPC_CALL_COMPLETE);
skb->destructor = NULL;
sp->call = NULL;
rxrpc_put_call(call);
rxrpc_free_skb(skb);
return 0;
}
/*
* handle an error received on the local endpoint
*/
void rxrpc_UDP_error_report(struct sock *sk)
{
struct sock_exterr_skb *serr;
struct rxrpc_transport *trans;
struct rxrpc_local *local = sk->sk_user_data;
struct rxrpc_peer *peer;
struct sk_buff *skb;
__be32 addr;
__be16 port;
_enter("%p{%d}", sk, local->debug_id);
skb = skb_dequeue(&sk->sk_error_queue);
if (!skb) {
_leave("UDP socket errqueue empty");
return;
}
rxrpc_new_skb(skb);
serr = SKB_EXT_ERR(skb);
addr = *(__be32 *)(skb_network_header(skb) + serr->addr_offset);
port = serr->port;
_net("Rx UDP Error from "NIPQUAD_FMT":%hu",
NIPQUAD(addr), ntohs(port));
_debug("Msg l:%d d:%d", skb->len, skb->data_len);
peer = rxrpc_find_peer(local, addr, port);
if (IS_ERR(peer)) {
rxrpc_free_skb(skb);
_leave(" [no peer]");
return;
}
trans = rxrpc_find_transport(local, peer);
if (!trans) {
rxrpc_put_peer(peer);
rxrpc_free_skb(skb);
_leave(" [no trans]");
return;
}
if (serr->ee.ee_origin == SO_EE_ORIGIN_ICMP &&
serr->ee.ee_type == ICMP_DEST_UNREACH &&
serr->ee.ee_code == ICMP_FRAG_NEEDED
) {
u32 mtu = serr->ee.ee_info;
_net("Rx Received ICMP Fragmentation Needed (%d)", mtu);
/* wind down the local interface MTU */
if (mtu > 0 && peer->if_mtu == 65535 && mtu < peer->if_mtu) {
peer->if_mtu = mtu;
_net("I/F MTU %u", mtu);
}
/* ip_rt_frag_needed() may have eaten the info */
if (mtu == 0)
mtu = ntohs(icmp_hdr(skb)->un.frag.mtu);
if (mtu == 0) {
/* they didn't give us a size, estimate one */
if (mtu > 1500) {
mtu >>= 1;
if (mtu < 1500)
mtu = 1500;
} else {
/*
* receive a message from an RxRPC socket
* - we need to be careful about two or more threads calling recvmsg
* simultaneously
*/
int rxrpc_recvmsg(struct kiocb *iocb, struct socket *sock,
struct msghdr *msg, size_t len, int flags)
{
struct rxrpc_skb_priv *sp;
struct rxrpc_call *call = NULL, *continue_call = NULL;
struct rxrpc_sock *rx = rxrpc_sk(sock->sk);
struct sk_buff *skb;
long timeo;
int copy, ret, ullen, offset, copied = 0;
u32 abort_code;
DEFINE_WAIT(wait);
_enter(",,,%zu,%d", len, flags);
if (flags & (MSG_OOB | MSG_TRUNC))
return -EOPNOTSUPP;
ullen = msg->msg_flags & MSG_CMSG_COMPAT ? 4 : sizeof(unsigned long);
timeo = sock_rcvtimeo(&rx->sk, flags & MSG_DONTWAIT);
msg->msg_flags |= MSG_MORE;
lock_sock(&rx->sk);
for (;;) {
/* return immediately if a client socket has no outstanding
* calls */
if (RB_EMPTY_ROOT(&rx->calls)) {
if (copied)
goto out;
if (rx->sk.sk_state != RXRPC_SERVER_LISTENING) {
release_sock(&rx->sk);
if (continue_call)
rxrpc_put_call(continue_call);
return -ENODATA;
}
}
/* get the next message on the Rx queue */
skb = skb_peek(&rx->sk.sk_receive_queue);
if (!skb) {
/* nothing remains on the queue */
if (copied &&
(msg->msg_flags & MSG_PEEK || timeo == 0))
goto out;
/* wait for a message to turn up */
release_sock(&rx->sk);
prepare_to_wait_exclusive(sk_sleep(&rx->sk), &wait,
TASK_INTERRUPTIBLE);
ret = sock_error(&rx->sk);
if (ret)
goto wait_error;
if (skb_queue_empty(&rx->sk.sk_receive_queue)) {
if (signal_pending(current))
goto wait_interrupted;
timeo = schedule_timeout(timeo);
}
finish_wait(sk_sleep(&rx->sk), &wait);
lock_sock(&rx->sk);
continue;
}
peek_next_packet:
sp = rxrpc_skb(skb);
call = sp->call;
ASSERT(call != NULL);
_debug("next pkt %s", rxrpc_pkts[sp->hdr.type]);
/* make sure we wait for the state to be updated in this call */
spin_lock_bh(&call->lock);
spin_unlock_bh(&call->lock);
if (test_bit(RXRPC_CALL_RELEASED, &call->flags)) {
_debug("packet from released call");
if (skb_dequeue(&rx->sk.sk_receive_queue) != skb)
BUG();
rxrpc_free_skb(skb);
continue;
}
/* determine whether to continue last data receive */
if (continue_call) {
_debug("maybe cont");
if (call != continue_call ||
skb->mark != RXRPC_SKB_MARK_DATA) {
release_sock(&rx->sk);
rxrpc_put_call(continue_call);
_leave(" = %d [noncont]", copied);
return copied;
}
}
rxrpc_get_call(call);
/* copy the peer address and timestamp */
if (!continue_call) {
if (msg->msg_name) {
size_t len =
sizeof(call->conn->trans->peer->srx);
memcpy(msg->msg_name,
&call->conn->trans->peer->srx, len);
msg->msg_namelen = len;
}
sock_recv_ts_and_drops(msg, &rx->sk, skb);
}
/* receive the message */
if (skb->mark != RXRPC_SKB_MARK_DATA)
goto receive_non_data_message;
_debug("recvmsg DATA #%u { %d, %d }",
ntohl(sp->hdr.seq), skb->len, sp->offset);
if (!continue_call) {
/* only set the control data once per recvmsg() */
ret = put_cmsg(msg, SOL_RXRPC, RXRPC_USER_CALL_ID,
ullen, &call->user_call_ID);
if (ret < 0)
goto copy_error;
ASSERT(test_bit(RXRPC_CALL_HAS_USERID, &call->flags));
}
ASSERTCMP(ntohl(sp->hdr.seq), >=, call->rx_data_recv);
ASSERTCMP(ntohl(sp->hdr.seq), <=, call->rx_data_recv + 1);
call->rx_data_recv = ntohl(sp->hdr.seq);
ASSERTCMP(ntohl(sp->hdr.seq), >, call->rx_data_eaten);
offset = sp->offset;
copy = skb->len - offset;
if (copy > len - copied)
copy = len - copied;
ret = skb_copy_datagram_iovec(skb, offset, msg->msg_iov, copy);
if (ret < 0)
goto copy_error;
/* handle piecemeal consumption of data packets */
_debug("copied %d+%d", copy, copied);
offset += copy;
copied += copy;
if (!(flags & MSG_PEEK))
sp->offset = offset;
if (sp->offset < skb->len) {
_debug("buffer full");
ASSERTCMP(copied, ==, len);
break;
}
/* we transferred the whole data packet */
if (sp->hdr.flags & RXRPC_LAST_PACKET) {
_debug("last");
if (call->conn->out_clientflag) {
/* last byte of reply received */
ret = copied;
goto terminal_message;
}
/* last bit of request received */
if (!(flags & MSG_PEEK)) {
_debug("eat packet");
if (skb_dequeue(&rx->sk.sk_receive_queue) !=
skb)
BUG();
rxrpc_free_skb(skb);
}
msg->msg_flags &= ~MSG_MORE;
break;
}
/* move on to the next data message */
_debug("next");
if (!continue_call)
continue_call = sp->call;
else
rxrpc_put_call(call);
call = NULL;
if (flags & MSG_PEEK) {
_debug("peek next");
skb = skb->next;
if (skb == (struct sk_buff *) &rx->sk.sk_receive_queue)
break;
goto peek_next_packet;
}
_debug("eat packet");
if (skb_dequeue(&rx->sk.sk_receive_queue) != skb)
BUG();
rxrpc_free_skb(skb);
}
void rxrpc_reject_packets(struct work_struct *work)
{
union {
struct sockaddr sa;
struct sockaddr_in sin;
} sa;
struct rxrpc_skb_priv *sp;
struct rxrpc_header hdr;
struct rxrpc_local *local;
struct sk_buff *skb;
struct msghdr msg;
struct kvec iov[2];
size_t size;
__be32 code;
local = container_of(work, struct rxrpc_local, rejecter);
rxrpc_get_local(local);
_enter("%d", local->debug_id);
iov[0].iov_base = &hdr;
iov[0].iov_len = sizeof(hdr);
iov[1].iov_base = &code;
iov[1].iov_len = sizeof(code);
size = sizeof(hdr) + sizeof(code);
msg.msg_name = &sa;
msg.msg_control = NULL;
msg.msg_controllen = 0;
msg.msg_flags = 0;
memset(&sa, 0, sizeof(sa));
sa.sa.sa_family = local->srx.transport.family;
switch (sa.sa.sa_family) {
case AF_INET:
msg.msg_namelen = sizeof(sa.sin);
break;
default:
msg.msg_namelen = 0;
break;
}
memset(&hdr, 0, sizeof(hdr));
hdr.type = RXRPC_PACKET_TYPE_ABORT;
while ((skb = skb_dequeue(&local->reject_queue))) {
sp = rxrpc_skb(skb);
switch (sa.sa.sa_family) {
case AF_INET:
sa.sin.sin_port = udp_hdr(skb)->source;
sa.sin.sin_addr.s_addr = ip_hdr(skb)->saddr;
code = htonl(skb->priority);
hdr.epoch = sp->hdr.epoch;
hdr.cid = sp->hdr.cid;
hdr.callNumber = sp->hdr.callNumber;
hdr.serviceId = sp->hdr.serviceId;
hdr.flags = sp->hdr.flags;
hdr.flags ^= RXRPC_CLIENT_INITIATED;
hdr.flags &= RXRPC_CLIENT_INITIATED;
kernel_sendmsg(local->socket, &msg, iov, 2, size);
break;
default:
break;
}
rxrpc_free_skb(skb);
rxrpc_put_local(local);
}
rxrpc_put_local(local);
_leave("");
}
int rxrpc_recvmsg(struct kiocb *iocb, struct socket *sock,
struct msghdr *msg, size_t len, int flags)
{
struct rxrpc_skb_priv *sp;
struct rxrpc_call *call = NULL, *continue_call = NULL;
struct rxrpc_sock *rx = rxrpc_sk(sock->sk);
struct sk_buff *skb;
long timeo;
int copy, ret, ullen, offset, copied = 0;
u32 abort_code;
DEFINE_WAIT(wait);
_enter(",,,%zu,%d", len, flags);
if (flags & (MSG_OOB | MSG_TRUNC))
return -EOPNOTSUPP;
ullen = msg->msg_flags & MSG_CMSG_COMPAT ? 4 : sizeof(unsigned long);
timeo = sock_rcvtimeo(&rx->sk, flags & MSG_DONTWAIT);
msg->msg_flags |= MSG_MORE;
lock_sock(&rx->sk);
for (;;) {
if (RB_EMPTY_ROOT(&rx->calls)) {
if (copied)
goto out;
if (rx->sk.sk_state != RXRPC_SERVER_LISTENING) {
release_sock(&rx->sk);
if (continue_call)
rxrpc_put_call(continue_call);
return -ENODATA;
}
}
skb = skb_peek(&rx->sk.sk_receive_queue);
if (!skb) {
if (copied &&
(msg->msg_flags & MSG_PEEK || timeo == 0))
goto out;
release_sock(&rx->sk);
prepare_to_wait_exclusive(sk_sleep(&rx->sk), &wait,
TASK_INTERRUPTIBLE);
ret = sock_error(&rx->sk);
if (ret)
goto wait_error;
if (skb_queue_empty(&rx->sk.sk_receive_queue)) {
if (signal_pending(current))
goto wait_interrupted;
timeo = schedule_timeout(timeo);
}
finish_wait(sk_sleep(&rx->sk), &wait);
lock_sock(&rx->sk);
continue;
}
peek_next_packet:
sp = rxrpc_skb(skb);
call = sp->call;
ASSERT(call != NULL);
_debug("next pkt %s", rxrpc_pkts[sp->hdr.type]);
spin_lock_bh(&call->lock);
spin_unlock_bh(&call->lock);
if (test_bit(RXRPC_CALL_RELEASED, &call->flags)) {
_debug("packet from released call");
if (skb_dequeue(&rx->sk.sk_receive_queue) != skb)
BUG();
rxrpc_free_skb(skb);
continue;
}
if (continue_call) {
_debug("maybe cont");
if (call != continue_call ||
skb->mark != RXRPC_SKB_MARK_DATA) {
release_sock(&rx->sk);
rxrpc_put_call(continue_call);
_leave(" = %d [noncont]", copied);
return copied;
}
}
rxrpc_get_call(call);
if (!continue_call) {
if (msg->msg_name && msg->msg_namelen > 0)
memcpy(msg->msg_name,
&call->conn->trans->peer->srx,
sizeof(call->conn->trans->peer->srx));
sock_recv_ts_and_drops(msg, &rx->sk, skb);
}
if (skb->mark != RXRPC_SKB_MARK_DATA)
goto receive_non_data_message;
_debug("recvmsg DATA #%u { %d, %d }",
ntohl(sp->hdr.seq), skb->len, sp->offset);
if (!continue_call) {
ret = put_cmsg(msg, SOL_RXRPC, RXRPC_USER_CALL_ID,
ullen, &call->user_call_ID);
if (ret < 0)
goto copy_error;
ASSERT(test_bit(RXRPC_CALL_HAS_USERID, &call->flags));
}
ASSERTCMP(ntohl(sp->hdr.seq), >=, call->rx_data_recv);
ASSERTCMP(ntohl(sp->hdr.seq), <=, call->rx_data_recv + 1);
call->rx_data_recv = ntohl(sp->hdr.seq);
ASSERTCMP(ntohl(sp->hdr.seq), >, call->rx_data_eaten);
offset = sp->offset;
copy = skb->len - offset;
if (copy > len - copied)
copy = len - copied;
if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
ret = skb_copy_datagram_iovec(skb, offset,
msg->msg_iov, copy);
} else {
ret = skb_copy_and_csum_datagram_iovec(skb, offset,
msg->msg_iov);
if (ret == -EINVAL)
goto csum_copy_error;
}
if (ret < 0)
goto copy_error;
_debug("copied %d+%d", copy, copied);
offset += copy;
copied += copy;
if (!(flags & MSG_PEEK))
sp->offset = offset;
if (sp->offset < skb->len) {
_debug("buffer full");
ASSERTCMP(copied, ==, len);
break;
}
if (sp->hdr.flags & RXRPC_LAST_PACKET) {
_debug("last");
if (call->conn->out_clientflag) {
ret = copied;
goto terminal_message;
}
if (!(flags & MSG_PEEK)) {
_debug("eat packet");
if (skb_dequeue(&rx->sk.sk_receive_queue) !=
skb)
BUG();
rxrpc_free_skb(skb);
}
msg->msg_flags &= ~MSG_MORE;
break;
}
_debug("next");
if (!continue_call)
continue_call = sp->call;
else
rxrpc_put_call(call);
call = NULL;
if (flags & MSG_PEEK) {
_debug("peek next");
skb = skb->next;
if (skb == (struct sk_buff *) &rx->sk.sk_receive_queue)
break;
goto peek_next_packet;
}
_debug("eat packet");
if (skb_dequeue(&rx->sk.sk_receive_queue) != skb)
BUG();
rxrpc_free_skb(skb);
}
/*
* Perform retransmission of NAK'd and unack'd packets.
*/
static void rxrpc_resend(struct rxrpc_call *call, ktime_t now)
{
struct rxrpc_skb_priv *sp;
struct sk_buff *skb;
rxrpc_seq_t cursor, seq, top;
ktime_t max_age, oldest, ack_ts;
int ix;
u8 annotation, anno_type, retrans = 0, unacked = 0;
_enter("{%d,%d}", call->tx_hard_ack, call->tx_top);
max_age = ktime_sub_ms(now, rxrpc_resend_timeout);
spin_lock_bh(&call->lock);
cursor = call->tx_hard_ack;
top = call->tx_top;
ASSERT(before_eq(cursor, top));
if (cursor == top)
goto out_unlock;
/* Scan the packet list without dropping the lock and decide which of
* the packets in the Tx buffer we're going to resend and what the new
* resend timeout will be.
*/
oldest = now;
for (seq = cursor + 1; before_eq(seq, top); seq++) {
ix = seq & RXRPC_RXTX_BUFF_MASK;
annotation = call->rxtx_annotations[ix];
anno_type = annotation & RXRPC_TX_ANNO_MASK;
annotation &= ~RXRPC_TX_ANNO_MASK;
if (anno_type == RXRPC_TX_ANNO_ACK)
continue;
skb = call->rxtx_buffer[ix];
rxrpc_see_skb(skb, rxrpc_skb_tx_seen);
sp = rxrpc_skb(skb);
if (anno_type == RXRPC_TX_ANNO_UNACK) {
if (ktime_after(skb->tstamp, max_age)) {
if (ktime_before(skb->tstamp, oldest))
oldest = skb->tstamp;
continue;
}
if (!(annotation & RXRPC_TX_ANNO_RESENT))
unacked++;
}
/* Okay, we need to retransmit a packet. */
call->rxtx_annotations[ix] = RXRPC_TX_ANNO_RETRANS | annotation;
retrans++;
trace_rxrpc_retransmit(call, seq, annotation | anno_type,
ktime_to_ns(ktime_sub(skb->tstamp, max_age)));
}
call->resend_at = ktime_add_ms(oldest, rxrpc_resend_timeout);
if (unacked)
rxrpc_congestion_timeout(call);
/* If there was nothing that needed retransmission then it's likely
* that an ACK got lost somewhere. Send a ping to find out instead of
* retransmitting data.
*/
if (!retrans) {
rxrpc_set_timer(call, rxrpc_timer_set_for_resend, now);
spin_unlock_bh(&call->lock);
ack_ts = ktime_sub(now, call->acks_latest_ts);
if (ktime_to_ns(ack_ts) < call->peer->rtt)
goto out;
rxrpc_propose_ACK(call, RXRPC_ACK_PING, 0, 0, true, false,
rxrpc_propose_ack_ping_for_lost_ack);
rxrpc_send_ack_packet(call, true);
goto out;
}
/* Now go through the Tx window and perform the retransmissions. We
* have to drop the lock for each send. If an ACK comes in whilst the
* lock is dropped, it may clear some of the retransmission markers for
* packets that it soft-ACKs.
*/
for (seq = cursor + 1; before_eq(seq, top); seq++) {
ix = seq & RXRPC_RXTX_BUFF_MASK;
annotation = call->rxtx_annotations[ix];
anno_type = annotation & RXRPC_TX_ANNO_MASK;
if (anno_type != RXRPC_TX_ANNO_RETRANS)
continue;
skb = call->rxtx_buffer[ix];
rxrpc_get_skb(skb, rxrpc_skb_tx_got);
spin_unlock_bh(&call->lock);
if (rxrpc_send_data_packet(call, skb, true) < 0) {
rxrpc_free_skb(skb, rxrpc_skb_tx_freed);
return;
}
if (rxrpc_is_client_call(call))
rxrpc_expose_client_call(call);
rxrpc_free_skb(skb, rxrpc_skb_tx_freed);
spin_lock_bh(&call->lock);
/* We need to clear the retransmit state, but there are two
* things we need to be aware of: A new ACK/NAK might have been
* received and the packet might have been hard-ACK'd (in which
* case it will no longer be in the buffer).
*/
if (after(seq, call->tx_hard_ack)) {
annotation = call->rxtx_annotations[ix];
anno_type = annotation & RXRPC_TX_ANNO_MASK;
if (anno_type == RXRPC_TX_ANNO_RETRANS ||
anno_type == RXRPC_TX_ANNO_NAK) {
annotation &= ~RXRPC_TX_ANNO_MASK;
annotation |= RXRPC_TX_ANNO_UNACK;
}
annotation |= RXRPC_TX_ANNO_RESENT;
call->rxtx_annotations[ix] = annotation;
}
if (after(call->tx_hard_ack, seq))
seq = call->tx_hard_ack;
}
out_unlock:
spin_unlock_bh(&call->lock);
out:
_leave("");
}
