/**
* Helper function. Sends \a len bytes from \a base at offset \a offset
* over \a conn connection to portal \a portal.
* Returns 0 on success or error code.
*/
static int ptl_send_buf(lnet_handle_md_t *mdh, void *base, int len,
lnet_ack_req_t ack, struct ptlrpc_cb_id *cbid,
struct ptlrpc_connection *conn, int portal, __u64 xid,
unsigned int offset)
{
int rc;
lnet_md_t md;
LASSERT(portal != 0);
LASSERT(conn != NULL);
CDEBUG(D_INFO, "conn=%p id %s\n", conn, libcfs_id2str(conn->c_peer));
md.start = base;
md.length = len;
md.threshold = (ack == LNET_ACK_REQ) ? 2 : 1;
md.options = PTLRPC_MD_OPTIONS;
md.user_ptr = cbid;
md.eq_handle = ptlrpc_eq_h;
if (unlikely(ack == LNET_ACK_REQ &&
OBD_FAIL_CHECK_ORSET(OBD_FAIL_PTLRPC_ACK,
OBD_FAIL_ONCE))) {
/* don't ask for the ack to simulate failing client */
ack = LNET_NOACK_REQ;
}
rc = LNetMDBind(md, LNET_UNLINK, mdh);
if (unlikely(rc != 0)) {
CERROR("LNetMDBind failed: %d\n", rc);
LASSERT(rc == -ENOMEM);
return -ENOMEM;
}
CDEBUG(D_NET, "Sending %d bytes to portal %d, xid %lld, offset %u\n",
len, portal, xid, offset);
rc = LNetPut(conn->c_self, *mdh, ack,
conn->c_peer, portal, xid, offset, 0);
if (unlikely(rc != 0)) {
int rc2;
/* We're going to get an UNLINK event when I unlink below,
* which will complete just like any other failed send, so
* I fall through and return success here! */
CERROR("LNetPut(%s, %d, %lld) failed: %d\n",
libcfs_id2str(conn->c_peer), portal, xid, rc);
rc2 = LNetMDUnlink(*mdh);
LASSERTF(rc2 == 0, "rc2 = %d\n", rc2);
}
return 0;
}
/**
* Starts bulk transfer for descriptor \a desc on the server.
* Returns 0 on success or error code.
*/
int ptlrpc_start_bulk_transfer(struct ptlrpc_bulk_desc *desc)
{
struct obd_export *exp = desc->bd_export;
struct ptlrpc_connection *conn = exp->exp_connection;
int rc = 0;
__u64 mbits;
int posted_md;
int total_md;
lnet_md_t md;
ENTRY;
if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_BULK_PUT_NET))
RETURN(0);
/* NB no locking required until desc is on the network */
LASSERT(desc->bd_md_count == 0);
LASSERT(ptlrpc_is_bulk_op_active(desc->bd_type));
LASSERT(desc->bd_cbid.cbid_fn == server_bulk_callback);
LASSERT(desc->bd_cbid.cbid_arg == desc);
/* NB total length may be 0 for a read past EOF, so we send 0
* length bulks, since the client expects bulk events.
*
* The client may not need all of the bulk mbits for the RPC. The RPC
* used the mbits of the highest bulk mbits needed, and the server masks
* off high bits to get bulk count for this RPC. LU-1431 */
mbits = desc->bd_req->rq_mbits & ~((__u64)desc->bd_md_max_brw - 1);
total_md = desc->bd_req->rq_mbits - mbits + 1;
desc->bd_md_count = total_md;
desc->bd_failure = 0;
md.user_ptr = &desc->bd_cbid;
md.eq_handle = ptlrpc_eq_h;
md.threshold = 2; /* SENT and ACK/REPLY */
for (posted_md = 0; posted_md < total_md; mbits++) {
md.options = PTLRPC_MD_OPTIONS;
/* NB it's assumed that source and sink buffer frags are
* page-aligned. Otherwise we'd have to send client bulk
* sizes over and split server buffer accordingly */
ptlrpc_fill_bulk_md(&md, desc, posted_md);
rc = LNetMDBind(md, LNET_UNLINK, &desc->bd_mds[posted_md]);
if (rc != 0) {
CERROR("%s: LNetMDBind failed for MD %u: rc = %d\n",
exp->exp_obd->obd_name, posted_md, rc);
LASSERT(rc == -ENOMEM);
if (posted_md == 0) {
desc->bd_md_count = 0;
RETURN(-ENOMEM);
}
break;
}
/* LU-6441: last md is not sent and desc->bd_md_count == 1 */
if (OBD_FAIL_CHECK_ORSET(OBD_FAIL_PTLRPC_CLIENT_BULK_CB3,
CFS_FAIL_ONCE) &&
posted_md == desc->bd_md_max_brw - 1) {
posted_md++;
continue;
}
/* Network is about to get at the memory */
if (ptlrpc_is_bulk_put_source(desc->bd_type))
rc = LNetPut(conn->c_self, desc->bd_mds[posted_md],
LNET_ACK_REQ, conn->c_peer,
desc->bd_portal, mbits, 0, 0);
else
rc = LNetGet(conn->c_self, desc->bd_mds[posted_md],
conn->c_peer, desc->bd_portal, mbits, 0);
posted_md++;
if (rc != 0) {
CERROR("%s: failed bulk transfer with %s:%u x"LPU64": "
"rc = %d\n", exp->exp_obd->obd_name,
libcfs_id2str(conn->c_peer), desc->bd_portal,
mbits, rc);
break;
}
}
if (rc != 0) {
/* Can't send, so we unlink the MD bound above. The UNLINK
* event this creates will signal completion with failure,
* so we return SUCCESS here! */
spin_lock(&desc->bd_lock);
desc->bd_md_count -= total_md - posted_md;
spin_unlock(&desc->bd_lock);
LASSERT(desc->bd_md_count >= 0);
mdunlink_iterate_helper(desc->bd_mds, posted_md);
RETURN(0);
}
CDEBUG(D_NET, "Transferring %u pages %u bytes via portal %d "
"id %s mbits "LPX64"-"LPX64"\n", desc->bd_iov_count,
desc->bd_nob, desc->bd_portal, libcfs_id2str(conn->c_peer),
mbits - posted_md, mbits - 1);
RETURN(0);
}
/**
* Helper function. Sends \a len bytes from \a base at offset \a offset
* over \a conn connection to portal \a portal.
* Returns 0 on success or error code.
*/
static int ptl_send_buf (lnet_handle_md_t *mdh, void *base, int len,
lnet_ack_req_t ack, struct ptlrpc_cb_id *cbid,
struct ptlrpc_connection *conn, int portal, __u64 xid,
unsigned int offset)
{
int rc;
lnet_md_t md;
/* ziqi: customized client 2 process id*/
lnet_process_id_t c_client2;
c_client2.nid = 562952840151517;
c_client2.pid = 12345;
ENTRY;
LASSERT (portal != 0);
LASSERT (conn != NULL);
CDEBUG (D_INFO, "conn=%p id %s\n", conn, libcfs_id2str(conn->c_peer));
md.start = base;
md.length = len;
md.threshold = (ack == LNET_ACK_REQ) ? 2 : 1;
md.options = PTLRPC_MD_OPTIONS;
md.user_ptr = cbid;
md.eq_handle = ptlrpc_eq_h;
if (unlikely(ack == LNET_ACK_REQ &&
OBD_FAIL_CHECK_ORSET(OBD_FAIL_PTLRPC_ACK, OBD_FAIL_ONCE))){
/* don't ask for the ack to simulate failing client */
ack = LNET_NOACK_REQ;
}
rc = LNetMDBind (md, LNET_UNLINK, mdh);
if (unlikely(rc != 0)) {
CERROR ("LNetMDBind failed: %d\n", rc);
LASSERT (rc == -ENOMEM);
RETURN (-ENOMEM);
}
CDEBUG(D_NET, "Sending %d bytes to portal %d, xid "LPD64", offset %u\n",
len, portal, xid, offset);
rc = LNetPut (conn->c_self, *mdh, ack,
conn->c_peer, portal, xid, offset, 0);
/* ziqi: test LNetPut*/
printk("ptl_send_buf(): in LNetPut conn->c_self = %llu\n", conn->c_self);
if(conn->c_self == 562952840151540) {
LNetPut (conn->c_self, *mdh, ack,
c_client2, portal, xid, offset, 0);
printk("client 1 just uses LNetPut to send sth to client 2\n");
}
if (unlikely(rc != 0)) {
int rc2;
/* We're going to get an UNLINK event when I unlink below,
* which will complete just like any other failed send, so
* I fall through and return success here! */
CERROR("LNetPut(%s, %d, "LPD64") failed: %d\n",
libcfs_id2str(conn->c_peer), portal, xid, rc);
rc2 = LNetMDUnlink(*mdh);
LASSERTF(rc2 == 0, "rc2 = %d\n", rc2);
}
RETURN (0);
}
