/**
* Server side bulk abort. Idempotent. Not thread-safe (i.e. only
* serialises with completion callback)
*/
void ptlrpc_abort_bulk(struct ptlrpc_bulk_desc *desc)
{
struct l_wait_info lwi;
int rc;
LASSERT(!in_interrupt()); /* might sleep */
if (!ptlrpc_server_bulk_active(desc)) /* completed or */
return; /* never started */
/* We used to poison the pages with 0xab here because we did not want to
* send any meaningful data over the wire for evicted clients (bug 9297)
* However, this is no longer safe now that we use the page cache on the
* OSS (bug 20560) */
/* The unlink ensures the callback happens ASAP and is the last
* one. If it fails, it must be because completion just happened,
* but we must still l_wait_event() in this case, to give liblustre
* a chance to run server_bulk_callback()*/
mdunlink_iterate_helper(desc->bd_mds, desc->bd_md_max_brw);
for (;;) {
/* Network access will complete in finite time but the HUGE
* timeout lets us CWARN for visibility of sluggish NALs */
lwi = LWI_TIMEOUT_INTERVAL(cfs_time_seconds(LONG_UNLINK),
cfs_time_seconds(1), NULL, NULL);
rc = l_wait_event(desc->bd_waitq,
!ptlrpc_server_bulk_active(desc), &lwi);
if (rc == 0)
return;
LASSERT(rc == -ETIMEDOUT);
CWARN("Unexpectedly long timeout: desc %p\n", desc);
}
}
/**
* Disconnect a bulk desc from the network. Idempotent. Not
* thread-safe (i.e. only interlocks with completion callback).
* Returns 1 on success or 0 if network unregistration failed for whatever
* reason.
*/
int ptlrpc_unregister_bulk(struct ptlrpc_request *req, int async)
{
struct ptlrpc_bulk_desc *desc = req->rq_bulk;
wait_queue_head_t *wq;
struct l_wait_info lwi;
int rc;
LASSERT(!in_interrupt()); /* might sleep */
/* Let's setup deadline for reply unlink. */
if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_LONG_BULK_UNLINK) &&
async && req->rq_bulk_deadline == 0 && cfs_fail_val == 0)
req->rq_bulk_deadline = ktime_get_real_seconds() + LONG_UNLINK;
if (ptlrpc_client_bulk_active(req) == 0) /* completed or */
return 1; /* never registered */
LASSERT(desc->bd_req == req); /* bd_req NULL until registered */
/* the unlink ensures the callback happens ASAP and is the last
* one. If it fails, it must be because completion just happened,
* but we must still l_wait_event() in this case to give liblustre
* a chance to run client_bulk_callback()
*/
mdunlink_iterate_helper(desc->bd_mds, desc->bd_md_max_brw);
if (ptlrpc_client_bulk_active(req) == 0) /* completed or */
return 1; /* never registered */
/* Move to "Unregistering" phase as bulk was not unlinked yet. */
ptlrpc_rqphase_move(req, RQ_PHASE_UNREG_BULK);
/* Do not wait for unlink to finish. */
if (async)
return 0;
if (req->rq_set)
wq = &req->rq_set->set_waitq;
else
wq = &req->rq_reply_waitq;
for (;;) {
/* Network access will complete in finite time but the HUGE
* timeout lets us CWARN for visibility of sluggish LNDs
*/
lwi = LWI_TIMEOUT_INTERVAL(cfs_time_seconds(LONG_UNLINK),
cfs_time_seconds(1), NULL, NULL);
rc = l_wait_event(*wq, !ptlrpc_client_bulk_active(req), &lwi);
if (rc == 0) {
ptlrpc_rqphase_move(req, req->rq_next_phase);
return 1;
}
LASSERT(rc == -ETIMEDOUT);
DEBUG_REQ(D_WARNING, req, "Unexpectedly long timeout: desc %p",
desc);
}
return 0;
}
/**
* rsx_bulkserver - Setup a source or sink for a server.
* @rq: RPC request associated with GET.
* @type: GET_SINK receive from client or PUT_SOURCE to push to a client.
* @ptl: portal to issue bulk xfer across.
* @iov: iovec array of receive buffer.
* @n: #iovecs.
* Returns: 0 or negative errno on error.
*/
int
rsx_bulkserver(struct pscrpc_request *rq, int type, int ptl,
struct iovec *iov, int n)
{
int sum, i, rc, comms_error;
struct pscrpc_bulk_desc *desc;
struct l_wait_info lwi;
uint64_t *v8;
uint8_t *v1;
psc_assert(type == BULK_GET_SINK || type == BULK_PUT_SOURCE);
desc = pscrpc_prep_bulk_exp(rq, n, type, ptl);
if (desc == NULL) {
psclog_warnx("pscrpc_prep_bulk_exp returned a null desc");
return (-ENOMEM); // XXX errno
}
desc->bd_nob = 0;
desc->bd_iov_count = n;
memcpy(desc->bd_iov, iov, n * sizeof(*iov));
for (i = 0; i < n; i++)
desc->bd_nob += iov[i].iov_len;
/* Check for client eviction during previous I/O before proceeding. */
if (desc->bd_export->exp_failed)
rc = -ENOTCONN;
else
rc = pscrpc_start_bulk_transfer(desc);
if (rc == 0) {
lwi = LWI_TIMEOUT_INTERVAL(OBD_TIMEOUT / 2,
100, pfl_rsx_timeout, desc);
rc = pscrpc_svr_wait_event(&desc->bd_waitq,
(!pscrpc_bulk_active(desc) || desc->bd_export->exp_failed),
&lwi, NULL);
LASSERT(rc == 0 || rc == -ETIMEDOUT);
if (rc == -ETIMEDOUT) {
DEBUG_REQ(PLL_ERROR, rq, "timeout on bulk GET");
pscrpc_abort_bulk(desc);
} else if (desc->bd_export->exp_failed) {
DEBUG_REQ(PLL_ERROR, rq, "eviction on bulk GET");
rc = -ENOTCONN;
pscrpc_abort_bulk(desc);
} else if (!desc->bd_success ||
desc->bd_nob_transferred != desc->bd_nob) {
DEBUG_REQ(PLL_ERROR, rq, "%s bulk GET %d(%d)",
desc->bd_success ? "truncated" : "network error on",
desc->bd_nob_transferred, desc->bd_nob);
rc = -ETIMEDOUT;
}
} else {
DEBUG_REQ(PLL_ERROR, rq, "pscrpc I/O bulk get failed: "
"rc=%d", rc);
}
comms_error = (rc != 0);
/* count the number of bytes received, and hold for later... */
if (rc == 0) {
v1 = desc->bd_iov[0].iov_base;
v8 = desc->bd_iov[0].iov_base;
if (v1 == NULL) {
DEBUG_REQ(PLL_ERROR, rq,
"desc->bd_iov[0].iov_base is NULL");
rc = -ENXIO;
goto out;
}
DEBUG_REQ(PLL_DIAG, rq,
"got %u bytes of bulk data across %d IOVs: "
"first byte is %#x (%"PRIx64")",
desc->bd_nob, desc->bd_iov_count, *v1, *v8);
sum = 0;
for (i = 0; i < desc->bd_iov_count; i++)
sum += desc->bd_iov[i].iov_len;
if (sum != desc->bd_nob)
DEBUG_REQ(PLL_WARN, rq,
"sum (%d) does not match bd_nob (%d)",
sum, desc->bd_nob);
//rc = pscrpc_reply(rq);
}
out:
if (rc == 0)
;
else if (!comms_error) {
/* Only reply if there were no comm problems with bulk. */
rq->rq_status = rc;
pscrpc_error(rq);
} else {
#if 0
// For now let's not free the reply state..
if (rq->rq_reply_state != NULL) {
/* reply out callback would free */
pscrpc_rs_decref(rq->rq_reply_state);
rq->rq_reply_state = NULL;
rq->rq_repmsg = NULL;
}
#endif
DEBUG_REQ(PLL_WARN, rq,
"ignoring bulk I/O comm error; "
"id %s - client will retry",
libcfs_id2str(rq->rq_peer));
}
pscrpc_free_bulk(desc);
return (rc);
}
