static void rds_iw_cm_fill_conn_param(struct rds_connection *conn, struct rdma_conn_param *conn_param, struct rds_iw_connect_private *dp, u32 protocol_version) { struct rds_iw_connection *ic = conn->c_transport_data; memset(conn_param, 0, sizeof(struct rdma_conn_param)); /* XXX tune these? */ conn_param->responder_resources = 1; conn_param->initiator_depth = 1; if (dp) { memset(dp, 0, sizeof(*dp)); dp->dp_saddr = conn->c_laddr; dp->dp_daddr = conn->c_faddr; dp->dp_protocol_major = RDS_PROTOCOL_MAJOR(protocol_version); dp->dp_protocol_minor = RDS_PROTOCOL_MINOR(protocol_version); dp->dp_protocol_minor_mask = cpu_to_be16(RDS_IW_SUPPORTED_PROTOCOLS); dp->dp_ack_seq = rds_iw_piggyb_ack(ic); /* Advertise flow control */ if (ic->i_flowctl) { unsigned int credits; credits = IB_GET_POST_CREDITS(atomic_read(&ic->i_credits)); dp->dp_credit = cpu_to_be32(credits); atomic_sub(IB_SET_POST_CREDITS(credits), &ic->i_credits); } conn_param->private_data = dp; conn_param->private_data_len = sizeof(*dp); } }
void rdsv3_ib_advertise_credits(struct rdsv3_connection *conn, unsigned int posted) { struct rdsv3_ib_connection *ic = conn->c_transport_data; RDSV3_DPRINTF4("rdsv3_ib_advertise_credits", "conn: %p, posted: %d", conn, posted); if (posted == 0) return; atomic_add_32(&ic->i_credits, IB_SET_POST_CREDITS(posted)); /* * Decide whether to send an update to the peer now. * If we would send a credit update for every single buffer we * post, we would end up with an ACK storm (ACK arrives, * consumes buffer, we refill the ring, send ACK to remote * advertising the newly posted buffer... ad inf) * * Performance pretty much depends on how often we send * credit updates - too frequent updates mean lots of ACKs. * Too infrequent updates, and the peer will run out of * credits and has to throttle. * For the time being, 16 seems to be a good compromise. */ if (IB_GET_POST_CREDITS(atomic_get(&ic->i_credits)) >= 16) set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); }
/* * This is the main function for allocating credits when sending * messages. * * Conceptually, we have two counters: * - send credits: this tells us how many WRs we're allowed * to submit without overruning the reciever's queue. For * each SEND WR we post, we decrement this by one. * * - posted credits: this tells us how many WRs we recently * posted to the receive queue. This value is transferred * to the peer as a "credit update" in a RDS header field. * Every time we transmit credits to the peer, we subtract * the amount of transferred credits from this counter. * * It is essential that we avoid situations where both sides have * exhausted their send credits, and are unable to send new credits * to the peer. We achieve this by requiring that we send at least * one credit update to the peer before exhausting our credits. * When new credits arrive, we subtract one credit that is withheld * until we've posted new buffers and are ready to transmit these * credits (see rdsv3_ib_send_add_credits below). * * The RDS send code is essentially single-threaded; rdsv3_send_xmit * grabs c_send_lock to ensure exclusive access to the send ring. * However, the ACK sending code is independent and can race with * message SENDs. * * In the send path, we need to update the counters for send credits * and the counter of posted buffers atomically - when we use the * last available credit, we cannot allow another thread to race us * and grab the posted credits counter. Hence, we have to use a * spinlock to protect the credit counter, or use atomics. * * Spinlocks shared between the send and the receive path are bad, * because they create unnecessary delays. An early implementation * using a spinlock showed a 5% degradation in throughput at some * loads. * * This implementation avoids spinlocks completely, putting both * counters into a single atomic, and updating that atomic using * atomic_add (in the receive path, when receiving fresh credits), * and using atomic_cmpxchg when updating the two counters. */ int rdsv3_ib_send_grab_credits(struct rdsv3_ib_connection *ic, uint32_t wanted, uint32_t *adv_credits, int need_posted) { unsigned int avail, posted, got = 0, advertise; long oldval, newval; RDSV3_DPRINTF4("rdsv3_ib_send_grab_credits", "ic: %p, %d %d %d", ic, wanted, *adv_credits, need_posted); *adv_credits = 0; if (!ic->i_flowctl) return (wanted); try_again: advertise = 0; oldval = newval = atomic_get(&ic->i_credits); posted = IB_GET_POST_CREDITS(oldval); avail = IB_GET_SEND_CREDITS(oldval); RDSV3_DPRINTF5("rdsv3_ib_send_grab_credits", "wanted (%u): credits=%u posted=%u\n", wanted, avail, posted); /* The last credit must be used to send a credit update. */ if (avail && !posted) avail--; if (avail < wanted) { struct rdsv3_connection *conn = ic->i_cm_id->context; /* Oops, there aren't that many credits left! */ set_bit(RDSV3_LL_SEND_FULL, &conn->c_flags); got = avail; } else { /* Sometimes you get what you want, lalala. */ got = wanted; } newval -= IB_SET_SEND_CREDITS(got); /* * If need_posted is non-zero, then the caller wants * the posted regardless of whether any send credits are * available. */ if (posted && (got || need_posted)) { advertise = min(posted, RDSV3_MAX_ADV_CREDIT); newval -= IB_SET_POST_CREDITS(advertise); } /* Finally bill everything */ if (atomic_cmpxchg(&ic->i_credits, oldval, newval) != oldval) goto try_again; *adv_credits = advertise; RDSV3_DPRINTF4("rdsv3_ib_send_grab_credits", "ic: %p, %d %d %d", ic, got, *adv_credits, need_posted); return (got); }
static void rdsv3_ib_cm_fill_conn_param(struct rdsv3_connection *conn, struct rdma_conn_param *conn_param, struct rdsv3_ib_connect_private *dp, uint32_t protocol_version, uint32_t max_responder_resources, uint32_t max_initiator_depth) { struct rdsv3_ib_connection *ic = conn->c_transport_data; struct rdsv3_ib_device *rds_ibdev; RDSV3_DPRINTF2("rdsv3_ib_cm_fill_conn_param", "Enter conn: %p conn_param: %p private: %p version: %d", conn, conn_param, dp, protocol_version); (void) memset(conn_param, 0, sizeof (struct rdma_conn_param)); rds_ibdev = ib_get_client_data(ic->i_cm_id->device, &rdsv3_ib_client); conn_param->responder_resources = MIN(rds_ibdev->max_responder_resources, max_responder_resources); conn_param->initiator_depth = MIN(rds_ibdev->max_initiator_depth, max_initiator_depth); conn_param->retry_count = min(rdsv3_ib_retry_count, 7); conn_param->rnr_retry_count = 7; if (dp) { (void) memset(dp, 0, sizeof (*dp)); dp->dp_saddr = conn->c_laddr; dp->dp_daddr = conn->c_faddr; dp->dp_protocol_major = RDS_PROTOCOL_MAJOR(protocol_version); dp->dp_protocol_minor = RDS_PROTOCOL_MINOR(protocol_version); dp->dp_protocol_minor_mask = htons(RDSV3_IB_SUPPORTED_PROTOCOLS); dp->dp_ack_seq = rdsv3_ib_piggyb_ack(ic); /* Advertise flow control */ if (ic->i_flowctl) { unsigned int credits; credits = IB_GET_POST_CREDITS( atomic_get(&ic->i_credits)); dp->dp_credit = htonl(credits); atomic_add_32(&ic->i_credits, -IB_SET_POST_CREDITS(credits)); } conn_param->private_data = dp; conn_param->private_data_len = sizeof (*dp); } RDSV3_DPRINTF2("rdsv3_ib_cm_fill_conn_param", "Return conn: %p conn_param: %p private: %p version: %d", conn, conn_param, dp, protocol_version); }
/* * This is the main function for allocating credits when sending * messages. * * Conceptually, we have two counters: * - send credits: this tells us how many WRs we're allowed * to submit without overruning the reciever's queue. For * each SEND WR we post, we decrement this by one. * * - posted credits: this tells us how many WRs we recently * posted to the receive queue. This value is transferred * to the peer as a "credit update" in a RDS header field. * Every time we transmit credits to the peer, we subtract * the amount of transferred credits from this counter. * * It is essential that we avoid situations where both sides have * exhausted their send credits, and are unable to send new credits * to the peer. We achieve this by requiring that we send at least * one credit update to the peer before exhausting our credits. * When new credits arrive, we subtract one credit that is withheld * until we've posted new buffers and are ready to transmit these * credits (see rds_ib_send_add_credits below). * * The RDS send code is essentially single-threaded; rds_send_xmit * grabs c_send_lock to ensure exclusive access to the send ring. * However, the ACK sending code is independent and can race with * message SENDs. * * In the send path, we need to update the counters for send credits * and the counter of posted buffers atomically - when we use the * last available credit, we cannot allow another thread to race us * and grab the posted credits counter. Hence, we have to use a * spinlock to protect the credit counter, or use atomics. * * Spinlocks shared between the send and the receive path are bad, * because they create unnecessary delays. An early implementation * using a spinlock showed a 5% degradation in throughput at some * loads. * * This implementation avoids spinlocks completely, putting both * counters into a single atomic, and updating that atomic using * atomic_add (in the receive path, when receiving fresh credits), * and using atomic_cmpxchg when updating the two counters. */ int rds_ib_send_grab_credits(struct rds_ib_connection *ic, u32 wanted, u32 *adv_credits, int need_posted, int max_posted) { unsigned int avail, posted, got = 0, advertise; long oldval, newval; *adv_credits = 0; if (!ic->i_flowctl) return wanted; try_again: advertise = 0; oldval = newval = atomic_read(&ic->i_credits); posted = IB_GET_POST_CREDITS(oldval); avail = IB_GET_SEND_CREDITS(oldval); rdsdebug("rds_ib_send_grab_credits(%u): credits=%u posted=%u\n", wanted, avail, posted); /* The last credit must be used to send a credit update. */ if (avail && !posted) avail--; if (avail < wanted) { struct rds_connection *conn = ic->i_cm_id->context; /* Oops, there aren't that many credits left! */ set_bit(RDS_LL_SEND_FULL, &conn->c_flags); got = avail; } else { /* Sometimes you get what you want, lalala. */ got = wanted; } newval -= IB_SET_SEND_CREDITS(got); /* * If need_posted is non-zero, then the caller wants * the posted regardless of whether any send credits are * available. */ if (posted && (got || need_posted)) { advertise = min_t(unsigned int, posted, max_posted); newval -= IB_SET_POST_CREDITS(advertise); }
static void rds_ib_cm_fill_conn_param(struct rds_connection *conn, struct rdma_conn_param *conn_param, struct rds_ib_connect_private *dp, u32 protocol_version, u32 max_responder_resources, u32 max_initiator_depth) { struct rds_ib_connection *ic = conn->c_transport_data; struct rds_ib_device *rds_ibdev = ic->rds_ibdev; memset(conn_param, 0, sizeof(struct rdma_conn_param)); conn_param->responder_resources = min_t(u32, rds_ibdev->max_responder_resources, max_responder_resources); conn_param->initiator_depth = min_t(u32, rds_ibdev->max_initiator_depth, max_initiator_depth); conn_param->retry_count = min_t(unsigned int, rds_ib_retry_count, 7); conn_param->rnr_retry_count = 7; if (dp) { memset(dp, 0, sizeof(*dp)); dp->dp_saddr = conn->c_laddr; dp->dp_daddr = conn->c_faddr; dp->dp_protocol_major = RDS_PROTOCOL_MAJOR(protocol_version); dp->dp_protocol_minor = RDS_PROTOCOL_MINOR(protocol_version); dp->dp_protocol_minor_mask = cpu_to_be16(RDS_IB_SUPPORTED_PROTOCOLS); dp->dp_ack_seq = cpu_to_be64(rds_ib_piggyb_ack(ic)); /* Advertise flow control */ if (ic->i_flowctl) { unsigned int credits; credits = IB_GET_POST_CREDITS(atomic_read(&ic->i_credits)); dp->dp_credit = cpu_to_be32(credits); atomic_sub(IB_SET_POST_CREDITS(credits), &ic->i_credits); } conn_param->private_data = dp; conn_param->private_data_len = sizeof(*dp); } }