/**
* Implementation of obd_ops::o_get_info.
*
* This function is not called from request handler, it is only used by
* direct call from nrs_orr_range_fill_physical() in ptlrpc, see LU-3239.
*
* \see ofd_get_info_hdl() for request handler function.
*
* \param[in] env execution environment
* \param[in] exp OBD export of OFD device
* \param[in] keylen length of \a key
* \param[in] key key name
* \param[out] vallen length of key value
* \param[out] val the key value to return
* \param[in] lsm not used in OFD
*
* \retval 0 if successful
* \retval negative value on error
*/
static int ofd_get_info(const struct lu_env *env, struct obd_export *exp,
__u32 keylen, void *key, __u32 *vallen, void *val,
struct lov_stripe_md *lsm)
{
struct ofd_thread_info *info;
struct ofd_device *ofd;
struct ll_fiemap_info_key *fm_key = key;
struct ll_user_fiemap *fiemap = val;
int rc = 0;
ENTRY;
if (exp->exp_obd == NULL) {
CDEBUG(D_IOCTL, "invalid client export %p\n", exp);
RETURN(-EINVAL);
}
ofd = ofd_exp(exp);
if (KEY_IS(KEY_FIEMAP)) {
info = ofd_info_init(env, exp);
rc = ostid_to_fid(&info->fti_fid, &fm_key->oa.o_oi,
ofd->ofd_lut.lut_lsd.lsd_osd_index);
if (rc != 0)
RETURN(rc);
rc = ofd_fiemap_get(env, ofd, &info->fti_fid, fiemap);
} else {
CERROR("%s: not supported key %s\n", ofd_name(ofd), (char*)key);
rc = -EOPNOTSUPP;
}
RETURN(rc);
}
/**
* Disconnect a connected client.
*
* This function terminates the client connection. The client export is
* disconnected (cleaned up) and client data on persistent storage is removed.
*
* \param[in] exp OBD export
*
* \retval 0 if successful
* \retval negative value on error
*/
int ofd_obd_disconnect(struct obd_export *exp)
{
struct ofd_device *ofd = ofd_exp(exp);
struct lu_env env;
int rc;
ENTRY;
LASSERT(exp);
class_export_get(exp);
if (!(exp->exp_flags & OBD_OPT_FORCE))
ofd_grant_sanity_check(ofd_obd(ofd), __FUNCTION__);
nodemap_del_member(exp);
rc = server_disconnect_export(exp);
ofd_grant_discard(exp);
/* Do not erase record for recoverable client. */
if (exp->exp_obd->obd_replayable &&
(!exp->exp_obd->obd_fail || exp->exp_failed)) {
rc = lu_env_init(&env, LCT_DT_THREAD);
if (rc)
GOTO(out, rc);
tgt_client_del(&env, exp);
lu_env_fini(&env);
}
out:
class_export_put(exp);
RETURN(rc);
}
/* Update last_rcvd records with the latest transaction data */
int ofd_txn_stop_cb(const struct lu_env *env, struct thandle *txn,
void *cookie)
{
struct ofd_device *ofd = cookie;
struct ofd_thread_info *info;
ENTRY;
info = lu_context_key_get(&env->le_ctx, &ofd_thread_key);
if (info->fti_exp == NULL)
RETURN(0);
LASSERT(ofd_exp(info->fti_exp) == ofd);
if (info->fti_has_trans) {
if (info->fti_mult_trans == 0) {
CERROR("More than one transaction "LPU64"\n",
info->fti_transno);
RETURN(0);
}
/* we need another transno to be assigned */
info->fti_transno = 0;
} else if (txn->th_result == 0) {
info->fti_has_trans = 1;
}
spin_lock(&ofd->ofd_lut.lut_translock);
if (txn->th_result != 0) {
if (info->fti_transno != 0) {
CERROR("Replay transno "LPU64" failed: rc %d\n",
info->fti_transno, txn->th_result);
info->fti_transno = 0;
}
} else if (info->fti_transno == 0) {
info->fti_transno = ++ofd->ofd_lut.lut_last_transno;
} else {
/* should be replay */
if (info->fti_transno > ofd->ofd_lut.lut_last_transno)
ofd->ofd_lut.lut_last_transno = info->fti_transno;
}
spin_unlock(&ofd->ofd_lut.lut_translock);
/** VBR: set new versions */
if (txn->th_result == 0 && info->fti_obj != NULL) {
dt_version_set(env, ofd_object_child(info->fti_obj),
info->fti_transno, txn);
info->fti_obj = NULL;
}
/* filling reply data */
CDEBUG(D_INODE, "transno = %llu, last_committed = %llu\n",
info->fti_transno, ofd_obd(ofd)->obd_last_committed);
/* if can't add callback, do sync write */
txn->th_sync |= !!tgt_last_commit_cb_add(txn, &ofd->ofd_lut,
info->fti_exp,
info->fti_transno);
return ofd_last_rcvd_update(info, txn);
}
/**
* Implementation of obd_ops::o_destroy_export.
*
* This function is called from class_export_destroy() to cleanup
* the OFD-specific data for export being destroyed.
*
* \param[in] exp OBD export
*
* \retval 0 if successful
* \retval negative value on error
*/
static int ofd_destroy_export(struct obd_export *exp)
{
struct ofd_device *ofd = ofd_exp(exp);
if (exp->exp_filter_data.fed_pending)
CERROR("%s: cli %s/%p has %lu pending on destroyed export"
"\n", exp->exp_obd->obd_name, exp->exp_client_uuid.uuid,
exp, exp->exp_filter_data.fed_pending);
target_destroy_export(exp);
if (unlikely(obd_uuid_equals(&exp->exp_obd->obd_uuid,
&exp->exp_client_uuid)))
return 0;
ldlm_destroy_export(exp);
tgt_client_free(exp);
ofd_fmd_cleanup(exp);
/*
* discard grants once we're sure no more
* interaction with the client is possible
*/
ofd_grant_discard(exp);
ofd_fmd_cleanup(exp);
if (exp_connect_flags(exp) & OBD_CONNECT_GRANT_SHRINK) {
if (ofd->ofd_tot_granted_clients > 0)
ofd->ofd_tot_granted_clients --;
}
if (!(exp->exp_flags & OBD_OPT_FORCE))
ofd_grant_sanity_check(exp->exp_obd, __FUNCTION__);
LASSERT(list_empty(&exp->exp_filter_data.fed_mod_list));
return 0;
}
/**
* Prepare bulk IO requests for processing.
*
* This function does initial checks of IO and calls corresponding
* functions for read/write processing.
*
* \param[in] env execution environment
* \param[in] cmd IO type (read/write)
* \param[in] exp OBD export of client
* \param[in] oa OBDO structure from request
* \param[in] objcount always 1
* \param[in] obj object data
* \param[in] rnb remote buffers
* \param[in] nr_local number of local buffers
* \param[in] lnb local buffers
*
* \retval 0 on successful prepare
* \retval negative value on error
*/
int ofd_preprw(const struct lu_env *env, int cmd, struct obd_export *exp,
struct obdo *oa, int objcount, struct obd_ioobj *obj,
struct niobuf_remote *rnb, int *nr_local,
struct niobuf_local *lnb)
{
struct tgt_session_info *tsi = tgt_ses_info(env);
struct ofd_device *ofd = ofd_exp(exp);
struct ofd_thread_info *info;
char *jobid;
const struct lu_fid *fid = &oa->o_oi.oi_fid;
int rc = 0;
if (*nr_local > PTLRPC_MAX_BRW_PAGES) {
CERROR("%s: bulk has too many pages %d, which exceeds the"
"maximum pages per RPC of %d\n",
exp->exp_obd->obd_name, *nr_local, PTLRPC_MAX_BRW_PAGES);
RETURN(-EPROTO);
}
if (tgt_ses_req(tsi) == NULL) { /* echo client case */
info = ofd_info_init(env, exp);
jobid = NULL;
} else {
info = tsi2ofd_info(tsi);
jobid = tsi->tsi_jobid;
}
LASSERT(oa != NULL);
if (OBD_FAIL_CHECK(OBD_FAIL_SRV_ENOENT)) {
struct ofd_seq *oseq;
oseq = ofd_seq_load(env, ofd, ostid_seq(&oa->o_oi));
if (IS_ERR(oseq)) {
CERROR("%s: Can not find seq for "DOSTID
": rc = %ld\n", ofd_name(ofd), POSTID(&oa->o_oi),
PTR_ERR(oseq));
RETURN(-EINVAL);
}
if (oseq->os_destroys_in_progress == 0) {
/* don't fail lookups for orphan recovery, it causes
* later LBUGs when objects still exist during
* precreate */
ofd_seq_put(env, oseq);
RETURN(-ENOENT);
}
ofd_seq_put(env, oseq);
}
LASSERT(objcount == 1);
LASSERT(obj->ioo_bufcnt > 0);
if (cmd == OBD_BRW_WRITE) {
la_from_obdo(&info->fti_attr, oa, OBD_MD_FLGETATTR);
rc = ofd_preprw_write(env, exp, ofd, fid, &info->fti_attr, oa,
objcount, obj, rnb, nr_local, lnb, jobid);
} else if (cmd == OBD_BRW_READ) {
ofd_grant_prepare_read(env, exp, oa);
rc = ofd_preprw_read(env, exp, ofd, fid, &info->fti_attr, oa,
obj->ioo_bufcnt, rnb, nr_local, lnb,
jobid);
obdo_from_la(oa, &info->fti_attr, LA_ATIME);
} else {
CERROR("%s: wrong cmd %d received!\n",
exp->exp_obd->obd_name, cmd);
rc = -EPROTO;
}
RETURN(rc);
}
/**
* Match client and OST server connection feature flags.
*
* Compute the compatibility flags for a connection request based on
* features mutually supported by client and server.
*
* The obd_export::exp_connect_data.ocd_connect_flags field in \a exp
* must not be updated here, otherwise a partially initialized value may
* be exposed. After the connection request is successfully processed,
* the top-level tgt_connect() request handler atomically updates the export
* connect flags from the obd_connect_data::ocd_connect_flags field of the
* reply. \see tgt_connect().
*
* \param[in] env execution environment
* \param[in] exp the obd_export associated with this
* client/target pair
* \param[in] data stores data for this connect request
* \param[in] new_connection is this connection new or not
*
* \retval 0 if success
* \retval -EPROTO client and server feature requirements are
* incompatible
* \retval -EBADF OST index in connect request doesn't match
* real OST index
*/
static int ofd_parse_connect_data(const struct lu_env *env,
struct obd_export *exp,
struct obd_connect_data *data,
bool new_connection)
{
struct ofd_device *ofd = ofd_exp(exp);
struct filter_export_data *fed = &exp->exp_filter_data;
if (!data)
RETURN(0);
CDEBUG(D_RPCTRACE, "%s: cli %s/%p ocd_connect_flags: "LPX64
" ocd_version: %x ocd_grant: %d ocd_index: %u"
" ocd_group %u\n",
exp->exp_obd->obd_name, exp->exp_client_uuid.uuid, exp,
data->ocd_connect_flags, data->ocd_version,
data->ocd_grant, data->ocd_index, data->ocd_group);
if (fed->fed_group != 0 && fed->fed_group != data->ocd_group) {
CWARN("!!! This export (nid %s) used object group %d "
"earlier; now it's trying to use group %d! This could "
"be a bug in the MDS. Please report to "
"https://jira.hpdd.intel.com/\n",
obd_export_nid2str(exp), fed->fed_group,
data->ocd_group);
RETURN(-EPROTO);
}
fed->fed_group = data->ocd_group;
data->ocd_connect_flags &= OST_CONNECT_SUPPORTED;
data->ocd_version = LUSTRE_VERSION_CODE;
/* Kindly make sure the SKIP_ORPHAN flag is from MDS. */
if (data->ocd_connect_flags & OBD_CONNECT_MDS)
CDEBUG(D_HA, "%s: Received MDS connection for group %u\n",
exp->exp_obd->obd_name, data->ocd_group);
else if (data->ocd_connect_flags & OBD_CONNECT_SKIP_ORPHAN)
RETURN(-EPROTO);
if (ofd_grant_param_supp(exp)) {
exp->exp_filter_data.fed_pagesize = data->ocd_blocksize;
/* ocd_{blocksize,inodespace} are log2 values */
data->ocd_blocksize = ofd->ofd_blockbits;
data->ocd_inodespace = ofd->ofd_dt_conf.ddp_inodespace;
/* ocd_grant_extent is in 1K blocks */
data->ocd_grant_extent = ofd->ofd_dt_conf.ddp_grant_frag >> 10;
}
if (data->ocd_connect_flags & OBD_CONNECT_GRANT)
data->ocd_grant = ofd_grant_connect(env, exp, data->ocd_grant,
new_connection);
if (data->ocd_connect_flags & OBD_CONNECT_INDEX) {
struct lr_server_data *lsd = &ofd->ofd_lut.lut_lsd;
int index = lsd->lsd_osd_index;
if (index != data->ocd_index) {
LCONSOLE_ERROR_MSG(0x136, "Connection from %s to index"
" %u doesn't match actual OST index"
" %u in last_rcvd file, bad "
"configuration?\n",
obd_export_nid2str(exp), index,
data->ocd_index);
RETURN(-EBADF);
}
if (!(lsd->lsd_feature_compat & OBD_COMPAT_OST)) {
/* this will only happen on the first connect */
lsd->lsd_feature_compat |= OBD_COMPAT_OST;
/* sync is not needed here as lut_client_add will
* set exp_need_sync flag */
tgt_server_data_update(env, &ofd->ofd_lut, 0);
}
}
if (OBD_FAIL_CHECK(OBD_FAIL_OST_BRW_SIZE)) {
data->ocd_brw_size = 65536;
} else if (data->ocd_connect_flags & OBD_CONNECT_BRW_SIZE) {
data->ocd_brw_size = min(data->ocd_brw_size,
(__u32)DT_MAX_BRW_SIZE);
if (data->ocd_brw_size == 0) {
CERROR("%s: cli %s/%p ocd_connect_flags: "LPX64
" ocd_version: %x ocd_grant: %d ocd_index: %u "
"ocd_brw_size is unexpectedly zero, "
"network data corruption?"
"Refusing connection of this client\n",
exp->exp_obd->obd_name,
exp->exp_client_uuid.uuid,
exp, data->ocd_connect_flags, data->ocd_version,
data->ocd_grant, data->ocd_index);
RETURN(-EPROTO);
}
}
if (data->ocd_connect_flags & OBD_CONNECT_CKSUM) {
__u32 cksum_types = data->ocd_cksum_types;
/* The client set in ocd_cksum_types the checksum types it
* supports. We have to mask off the algorithms that we don't
* support */
data->ocd_cksum_types &= cksum_types_supported_server();
if (unlikely(data->ocd_cksum_types == 0)) {
CERROR("%s: Connect with checksum support but no "
"ocd_cksum_types is set\n",
exp->exp_obd->obd_name);
RETURN(-EPROTO);
}
CDEBUG(D_RPCTRACE, "%s: cli %s supports cksum type %x, return "
"%x\n", exp->exp_obd->obd_name, obd_export_nid2str(exp),
cksum_types, data->ocd_cksum_types);
} else {
/* This client does not support OBD_CONNECT_CKSUM
* fall back to CRC32 */
CDEBUG(D_RPCTRACE, "%s: cli %s does not support "
"OBD_CONNECT_CKSUM, CRC32 will be used\n",
exp->exp_obd->obd_name, obd_export_nid2str(exp));
}
if (data->ocd_connect_flags & OBD_CONNECT_MAXBYTES)
data->ocd_maxbytes = ofd->ofd_dt_conf.ddp_maxbytes;
if (OCD_HAS_FLAG(data, PINGLESS)) {
if (ptlrpc_pinger_suppress_pings()) {
spin_lock(&exp->exp_obd->obd_dev_lock);
list_del_init(&exp->exp_obd_chain_timed);
spin_unlock(&exp->exp_obd->obd_dev_lock);
} else {
data->ocd_connect_flags &= ~OBD_CONNECT_PINGLESS;
}
}
RETURN(0);
}
/*
* last_rcvd & last_committed update callbacks
*/
static int ofd_last_rcvd_update(struct ofd_thread_info *info,
struct thandle *th)
{
struct ofd_device *ofd = ofd_exp(info->fti_exp);
struct filter_export_data *fed;
struct lsd_client_data *lcd;
__s32 rc = th->th_result;
__u64 *transno_p;
loff_t off;
int err;
bool lw_client = false;
ENTRY;
LASSERT(ofd);
LASSERT(info->fti_exp);
if (exp_connect_flags(info->fti_exp) & OBD_CONNECT_LIGHTWEIGHT)
lw_client = true;
fed = &info->fti_exp->exp_filter_data;
LASSERT(fed);
lcd = fed->fed_ted.ted_lcd;
/* if the export has already been disconnected, we have no last_rcvd
* slot, update server data with latest transno then */
if (lcd == NULL) {
CWARN("commit transaction for disconnected client %s: rc %d\n",
info->fti_exp->exp_client_uuid.uuid, rc);
err = tgt_server_data_write(info->fti_env, &ofd->ofd_lut, th);
RETURN(err);
}
/* ofd connect may cause transaction before export has last_rcvd
* slot */
if (fed->fed_ted.ted_lr_idx < 0 && !lw_client)
RETURN(0);
off = fed->fed_ted.ted_lr_off;
transno_p = &lcd->lcd_last_transno;
lcd->lcd_last_xid = info->fti_xid;
/*
* When we store zero transno in mcd we can lost last transno value
* because mcd contains 0, but msd is not yet written
* The server data should be updated also if the latest
* transno is rewritten by zero. See the bug 11125 for details.
*/
if (info->fti_transno == 0 &&
*transno_p == ofd->ofd_lut.lut_last_transno) {
spin_lock(&ofd->ofd_lut.lut_translock);
ofd->ofd_lut.lut_lsd.lsd_last_transno =
ofd->ofd_lut.lut_last_transno;
spin_unlock(&ofd->ofd_lut.lut_translock);
tgt_server_data_write(info->fti_env, &ofd->ofd_lut, th);
}
*transno_p = info->fti_transno;
if (lw_client) {
/* Although lightweight (LW) connections have no slot in
* last_rcvd, we still want to maintain the in-memory
* lsd_client_data structure in order to properly handle reply
* reconstruction. */
struct lu_target *tg =&ofd->ofd_lut;
bool update = false;
err = 0;
/* All operations performed by LW clients are synchronous and
* we store the committed transno in the last_rcvd header */
spin_lock(&tg->lut_translock);
if (info->fti_transno > tg->lut_lsd.lsd_last_transno) {
tg->lut_lsd.lsd_last_transno = info->fti_transno;
update = true;
}
spin_unlock(&tg->lut_translock);
if (update)
err = tgt_server_data_write(info->fti_env, tg, th);
} else {
LASSERT(fed->fed_ted.ted_lr_off > 0);
err = tgt_client_data_write(info->fti_env, &ofd->ofd_lut, lcd,
&off, th);
}
RETURN(err);
}
