/*
* Look-up a slave index file.
*
* \param env - is the environment passed by the caller
* \param dev - is the backend dt_device where to look-up/create the slave index
* \param parent - is the parent directory where to lookup the slave index
* \param glb_fid - is the fid of the global index file associated with this
* slave index.
* \param uuid - is the uuid of slave which is (re)connecting to the master
* target
*
* \retval - pointer to the dt_object of the slave index on success,
* appropriate error on failure
*/
struct dt_object *lquota_disk_slv_find(const struct lu_env *env,
struct dt_device *dev,
struct dt_object *parent,
const struct lu_fid *glb_fid,
struct obd_uuid *uuid)
{
struct lquota_thread_info *qti = lquota_info(env);
struct dt_object *slv_idx;
int rc;
ENTRY;
LASSERT(uuid != NULL);
CDEBUG(D_QUOTA, "lookup slave index file for %s\n",
obd_uuid2str(uuid));
/* generate filename associated with the slave */
rc = lquota_disk_slv_filename(glb_fid, uuid, qti->qti_buf);
if (rc)
RETURN(ERR_PTR(rc));
/* lookup slave index file */
rc = dt_lookup_dir(env, parent, qti->qti_buf, &qti->qti_fid);
if (rc)
RETURN(ERR_PTR(rc));
/* name is found, get the object */
slv_idx = dt_locate(env, dev, &qti->qti_fid);
if (IS_ERR(slv_idx))
RETURN(slv_idx);
if (slv_idx->do_index_ops == NULL) {
rc = slv_idx->do_ops->do_index_try(env, slv_idx,
&dt_quota_slv_features);
if (rc) {
CERROR("%s: failed to setup slave index operations for "
"%s, rc:%d\n", dev->dd_lu_dev.ld_obd->obd_name,
obd_uuid2str(uuid), rc);
lu_object_put(env, &slv_idx->do_lu);
slv_idx = ERR_PTR(rc);
}
}
RETURN(slv_idx);
}
static int lov_cl_add_target(const struct lu_env *env, struct lu_device *dev,
__u32 index)
{
struct obd_device *obd = dev->ld_obd;
struct lov_device *ld = lu2lov_dev(dev);
struct lov_tgt_desc *tgt;
struct lovsub_device *lsd;
struct cl_device *cl;
int rc;
obd_getref(obd);
tgt = obd->u.lov.lov_tgts[index];
LASSERT(tgt != NULL);
LASSERT(tgt->ltd_obd != NULL);
if (!tgt->ltd_obd->obd_set_up) {
CERROR("Target %s not set up\n", obd_uuid2str(&tgt->ltd_uuid));
return -EINVAL;
}
rc = lov_expand_targets(env, ld);
if (rc == 0 && ld->ld_flags & LOV_DEV_INITIALIZED) {
LASSERT(dev->ld_site != NULL);
cl = cl_type_setup(env, dev->ld_site, &lovsub_device_type,
tgt->ltd_obd->obd_lu_dev);
if (!IS_ERR(cl)) {
lsd = cl2lovsub_dev(cl);
lsd->acid_idx = index;
lsd->acid_super = ld;
ld->ld_target[index] = lsd;
} else {
CERROR("add failed (%d), deleting %s\n", rc,
obd_uuid2str(&tgt->ltd_uuid));
lov_cl_del_target(env, dev, index);
rc = PTR_ERR(cl);
}
}
obd_putref(obd);
return rc;
}
int lprocfs_exp_print_uuid_seq(cfs_hash_t *hs, cfs_hash_bd_t *bd,
struct hlist_node *hnode, void *cb_data)
{
struct obd_export *exp = cfs_hash_object(hs, hnode);
struct seq_file *m = cb_data;
if (exp->exp_nid_stats)
seq_printf(m, "%s\n", obd_uuid2str(&exp->exp_client_uuid));
return 0;
}
/*
* Look-up a slave index file. If the slave index isn't found:
* - if local is set to false, we allocate a FID from FID_SEQ_QUOTA sequence and
* create the index.
* - otherwise, we create the index file with a local reserved FID (see
* lquota_local_oid)
*
* \param env - is the environment passed by the caller
* \param dev - is the backend dt_device where to look-up/create the slave index
* \param parent - is the parent directory where to create the slave index if
* it does not exist already
* \param glb_fid - is the fid of the global index file associated with this
* slave index.
* \param uuid - is the uuid of slave which is (re)connecting to the master
* target
* \param local - indicate whether to use local reserved FID (LQUOTA_USR_OID
* & LQUOTA_GRP_OID) for the slave index creation or to
* allocate a new fid from sequence FID_SEQ_QUOTA
*
* \retval - pointer to the dt_object of the slave index on success,
* appropriate error on failure
*/
struct dt_object *lquota_disk_slv_find_create(const struct lu_env *env,
struct dt_device *dev,
struct dt_object *parent,
struct lu_fid *glb_fid,
struct obd_uuid *uuid,
bool local)
{
struct lquota_thread_info *qti = lquota_info(env);
struct dt_object *slv_idx;
int rc;
ENTRY;
LASSERT(uuid != NULL);
CDEBUG(D_QUOTA, "lookup/create slave index file for %s\n",
obd_uuid2str(uuid));
/* generate filename associated with the slave */
rc = lquota_disk_slv_filename(glb_fid, uuid, qti->qti_buf);
if (rc)
RETURN(ERR_PTR(rc));
/* Slave indexes uses the FID_SEQ_QUOTA sequence since they can be read
* through the network */
qti->qti_fid.f_seq = FID_SEQ_QUOTA;
qti->qti_fid.f_ver = 0;
if (local) {
int type;
rc = lquota_extract_fid(glb_fid, NULL, NULL, &type);
if (rc)
RETURN(ERR_PTR(rc));
/* use predefined fid in the reserved oid list */
qti->qti_fid.f_oid = (type == USRQUOTA) ? LQUOTA_USR_OID
: LQUOTA_GRP_OID;
slv_idx = local_index_find_or_create_with_fid(env, dev,
&qti->qti_fid,
parent,
qti->qti_buf,
LQUOTA_MODE,
&dt_quota_slv_features);
} else {
/* allocate fid dynamically if index does not exist already */
qti->qti_fid.f_oid = LQUOTA_GENERATED_OID;
/* lookup/create slave index file */
slv_idx = lquota_disk_find_create(env, dev, parent,
&qti->qti_fid,
&dt_quota_slv_features,
qti->qti_buf);
}
if (IS_ERR(slv_idx))
RETURN(slv_idx);
/* install index operation vector */
if (slv_idx->do_index_ops == NULL) {
rc = slv_idx->do_ops->do_index_try(env, slv_idx,
&dt_quota_slv_features);
if (rc) {
CERROR("%s: failed to setup index operations for "DFID
" rc:%d\n", dev->dd_lu_dev.ld_obd->obd_name,
PFID(lu_object_fid(&slv_idx->do_lu)), rc);
lu_object_put(env, &slv_idx->do_lu);
slv_idx = ERR_PTR(rc);
}
}
RETURN(slv_idx);
}
/*
* Handle quota request from slave.
*
* \param env - is the environment passed by the caller
* \param ld - is the lu device associated with the qmt
* \param req - is the quota acquire request
*/
static int qmt_dqacq(const struct lu_env *env, struct lu_device *ld,
struct ptlrpc_request *req)
{
struct qmt_device *qmt = lu2qmt_dev(ld);
struct quota_body *qbody, *repbody;
struct obd_uuid *uuid;
struct ldlm_lock *lock;
struct lquota_entry *lqe;
int pool_id, pool_type, qtype;
int rc;
ENTRY;
qbody = req_capsule_client_get(&req->rq_pill, &RMF_QUOTA_BODY);
if (qbody == NULL)
RETURN(err_serious(-EPROTO));
repbody = req_capsule_server_get(&req->rq_pill, &RMF_QUOTA_BODY);
if (repbody == NULL)
RETURN(err_serious(-EFAULT));
/* verify if global lock is stale */
if (!lustre_handle_is_used(&qbody->qb_glb_lockh))
RETURN(-ENOLCK);
lock = ldlm_handle2lock(&qbody->qb_glb_lockh);
if (lock == NULL)
RETURN(-ENOLCK);
LDLM_LOCK_PUT(lock);
uuid = &req->rq_export->exp_client_uuid;
if (req_is_rel(qbody->qb_flags) + req_is_acq(qbody->qb_flags) +
req_is_preacq(qbody->qb_flags) > 1) {
CERROR("%s: malformed quota request with conflicting flags set "
"(%x) from slave %s\n", qmt->qmt_svname,
qbody->qb_flags, obd_uuid2str(uuid));
RETURN(-EPROTO);
}
if (req_is_acq(qbody->qb_flags) || req_is_preacq(qbody->qb_flags)) {
/* acquire and pre-acquire should use a valid ID lock */
if (!lustre_handle_is_used(&qbody->qb_lockh))
RETURN(-ENOLCK);
lock = ldlm_handle2lock(&qbody->qb_lockh);
if (lock == NULL)
/* no lock associated with this handle */
RETURN(-ENOLCK);
LDLM_DEBUG(lock, "%sacquire request",
req_is_preacq(qbody->qb_flags) ? "pre" : "");
if (!obd_uuid_equals(&lock->l_export->exp_client_uuid, uuid)) {
/* sorry, no way to cheat ... */
LDLM_LOCK_PUT(lock);
RETURN(-ENOLCK);
}
if ((lock->l_flags & LDLM_FL_AST_SENT) != 0) {
struct ptlrpc_service_part *svc;
unsigned int timeout;
svc = req->rq_rqbd->rqbd_svcpt;
timeout = at_est2timeout(at_get(&svc->scp_at_estimate));
timeout = max(timeout, ldlm_timeout);
/* lock is being cancelled, prolong timeout */
ldlm_refresh_waiting_lock(lock, timeout);
}
LDLM_LOCK_PUT(lock);
}
/* extract pool & quota information from global index FID packed in the
* request */
rc = lquota_extract_fid(&qbody->qb_fid, &pool_id, &pool_type, &qtype);
if (rc)
RETURN(-EINVAL);
/* Find the quota entry associated with the quota id */
lqe = qmt_pool_lqe_lookup(env, qmt, pool_id, pool_type, qtype,
&qbody->qb_id);
if (IS_ERR(lqe))
RETURN(PTR_ERR(lqe));
/* process quota request */
rc = qmt_dqacq0(env, lqe, qmt, uuid, qbody->qb_flags, qbody->qb_count,
qbody->qb_usage, repbody);
if (lustre_handle_is_used(&qbody->qb_lockh))
/* return current qunit value only to slaves owning an per-ID
* quota lock. For enqueue, the qunit value will be returned in
* the LVB */
repbody->qb_qunit = lqe->lqe_qunit;
lqe_putref(lqe);
RETURN(rc);
}
/*
* Helper function to handle quota request from slave.
*
* \param env - is the environment passed by the caller
* \param lqe - is the lquota_entry subject to the quota request
* \param qmt - is the master device
* \param uuid - is the uuid associated with the slave
* \param qb_flags - are the quota request flags as packed in the quota_body
* \param qb_count - is the amount of quota space the slave wants to
* acquire/release
* \param qb_usage - is the current space usage on the slave
* \param repbody - is the quota_body of reply
*
* \retval 0 : success
* \retval -EDQUOT : out of quota
* -EINPROGRESS : inform client to retry write/create
* -ve : other appropriate errors
*/
int qmt_dqacq0(const struct lu_env *env, struct lquota_entry *lqe,
struct qmt_device *qmt, struct obd_uuid *uuid, __u32 qb_flags,
__u64 qb_count, __u64 qb_usage, struct quota_body *repbody)
{
struct qmt_thread_info *qti = qmt_info(env);
__u64 now, count;
struct dt_object *slv_obj = NULL;
__u64 slv_granted, slv_granted_bck;
struct thandle *th = NULL;
int rc, ret;
ENTRY;
LASSERT(uuid != NULL);
/* initialize reply */
memset(repbody, 0, sizeof(*repbody));
memcpy(&repbody->qb_id, &lqe->lqe_id, sizeof(repbody->qb_id));
if (OBD_FAIL_CHECK(OBD_FAIL_QUOTA_RECOVERABLE_ERR))
RETURN(-cfs_fail_val);
/* look-up index file associated with acquiring slave */
slv_obj = lquota_disk_slv_find(env, qmt->qmt_child, LQE_ROOT(lqe),
lu_object_fid(&LQE_GLB_OBJ(lqe)->do_lu),
uuid);
if (IS_ERR(slv_obj))
GOTO(out, rc = PTR_ERR(slv_obj));
/* pack slave fid in reply just for sanity check */
memcpy(&repbody->qb_slv_fid, lu_object_fid(&slv_obj->do_lu),
sizeof(struct lu_fid));
/* allocate & start transaction with enough credits to update
* global & slave indexes */
th = qmt_trans_start_with_slv(env, lqe, slv_obj, &qti->qti_restore);
if (IS_ERR(th))
GOTO(out, rc = PTR_ERR(th));
lqe_write_lock(lqe);
LQUOTA_DEBUG(lqe, "dqacq starts uuid:%s flags:0x%x wanted:"LPU64
" usage:"LPU64, obd_uuid2str(uuid), qb_flags, qb_count,
qb_usage);
/* Legal race, limits have been removed on master, but slave didn't
* receive the change yet. Just return EINPROGRESS until the slave gets
* notified. */
if (!lqe->lqe_enforced && !req_is_rel(qb_flags))
GOTO(out_locked, rc = -ESRCH);
/* recompute qunit in case it was never initialized */
qmt_revalidate(env, lqe);
/* slave just wants to acquire per-ID lock */
if (req_is_acq(qb_flags) && qb_count == 0)
GOTO(out_locked, rc = 0);
/* fetch how much quota space is already granted to this slave */
rc = qmt_slv_read(env, lqe, slv_obj, &slv_granted);
if (rc) {
LQUOTA_ERROR(lqe, "Failed to get granted for slave %s, rc=%d",
obd_uuid2str(uuid), rc);
GOTO(out_locked, rc);
}
/* recall how much space this slave currently owns in order to restore
* it in case of failure */
slv_granted_bck = slv_granted;
/* record current time for soft limit & grace time management */
now = (__u64)cfs_time_current_sec();
if (req_is_rel(qb_flags)) {
/* Slave would like to release quota space */
if (slv_granted < qb_count ||
lqe->lqe_granted < qb_count) {
/* can't release more than granted */
LQUOTA_ERROR(lqe, "Release too much! uuid:%s release:"
LPU64" granted:"LPU64", total:"LPU64,
obd_uuid2str(uuid), qb_count,
slv_granted, lqe->lqe_granted);
GOTO(out_locked, rc = -EINVAL);
}
repbody->qb_count = qb_count;
/* put released space back to global pool */
QMT_REL(lqe, slv_granted, qb_count);
GOTO(out_write, rc = 0);
}
if (req_has_rep(qb_flags) && slv_granted < qb_usage) {
/* Slave is reporting space usage in quota request and it turns
* out to be using more quota space than owned, so we adjust
* granted space regardless of the current state of affairs */
repbody->qb_count = qb_usage - slv_granted;
QMT_GRANT(lqe, slv_granted, repbody->qb_count);
}
if (!req_is_acq(qb_flags) && !req_is_preacq(qb_flags))
GOTO(out_write, rc = 0);
qmt_adjust_edquot(lqe, now);
if (lqe->lqe_edquot)
/* no hope to claim further space back */
GOTO(out_write, rc = -EDQUOT);
if (qmt_space_exhausted(lqe, now)) {
/* might have some free space once rebalancing is completed */
rc = req_is_acq(qb_flags) ? -EINPROGRESS : -EDQUOT;
GOTO(out_write, rc);
}
if (req_is_preacq(qb_flags)) {
/* slave would like to pre-acquire quota space. To do so, it
* reports in qb_count how much spare quota space it owns and we
* can grant back quota space which is consistent with qunit
* value. */
if (qb_count >= lqe->lqe_qunit)
/* slave already own the maximum it should */
GOTO(out_write, rc = 0);
count = qmt_alloc_expand(lqe, slv_granted, qb_count);
if (count == 0)
GOTO(out_write, rc = -EDQUOT);
repbody->qb_count += count;
QMT_GRANT(lqe, slv_granted, count);
GOTO(out_write, rc = 0);
}
/* processing acquire request with clients waiting */
if (lqe->lqe_hardlimit != 0 &&
lqe->lqe_granted + qb_count > lqe->lqe_hardlimit) {
/* cannot grant as much as asked, but can still afford to grant
* some quota space back */
count = lqe->lqe_hardlimit - lqe->lqe_granted;
repbody->qb_count += count;
QMT_GRANT(lqe, slv_granted, count);
GOTO(out_write, rc = 0);
}
/* Whouhou! we can satisfy the slave request! */
repbody->qb_count += qb_count;
QMT_GRANT(lqe, slv_granted, qb_count);
/* Try to expand the acquired count for DQACQ */
count = qmt_alloc_expand(lqe, slv_granted, 0);
if (count != 0) {
/* can even grant more than asked, it is like xmas ... */
repbody->qb_count += count;
QMT_GRANT(lqe, slv_granted, count);
GOTO(out_write, rc = 0);
}
GOTO(out_write, rc = 0);
out_write:
if (repbody->qb_count == 0)
GOTO(out_locked, rc);
/* start/stop grace timer if required */
if (lqe->lqe_softlimit != 0) {
if (lqe->lqe_granted > lqe->lqe_softlimit &&
lqe->lqe_gracetime == 0)
/* first time over soft limit, let's start grace
* timer */
lqe->lqe_gracetime = now + qmt_lqe_grace(lqe);
else if (lqe->lqe_granted <= lqe->lqe_softlimit &&
lqe->lqe_gracetime != 0)
/* Clear grace timer */
lqe->lqe_gracetime = 0;
}
/* Update slave index first since it is easier to roll back */
ret = qmt_slv_write(env, th, lqe, slv_obj, LQUOTA_BUMP_VER,
&repbody->qb_slv_ver, slv_granted);
if (ret) {
/* restore initial quota settings */
qmt_restore(lqe, &qti->qti_restore);
/* reset qb_count */
repbody->qb_count = 0;
GOTO(out_locked, rc = ret);
}
/* Update global index, no version bump needed */
ret = qmt_glb_write(env, th, lqe, 0, NULL);
if (ret) {
rc = ret;
/* restore initial quota settings */
qmt_restore(lqe, &qti->qti_restore);
/* reset qb_count */
repbody->qb_count = 0;
/* restore previous granted value */
ret = qmt_slv_write(env, th, lqe, slv_obj, 0, NULL,
slv_granted_bck);
if (ret) {
LQUOTA_ERROR(lqe, "failed to restore initial slave "
"value rc:%d ret%d", rc, ret);
LBUG();
}
qmt_adjust_edquot(lqe, now);
GOTO(out_locked, rc);
}
/* Total granted has been changed, let's try to adjust the qunit
* size according to the total granted & limits. */
qmt_adjust_qunit(env, lqe);
/* clear/set edquot flag and notify slaves via glimpse if needed */
qmt_adjust_edquot(lqe, now);
out_locked:
LQUOTA_DEBUG(lqe, "dqacq ends count:"LPU64" ver:"LPU64" rc:%d",
repbody->qb_count, repbody->qb_slv_ver, rc);
lqe_write_unlock(lqe);
out:
if (th != NULL && !IS_ERR(th))
dt_trans_stop(env, qmt->qmt_child, th);
if (slv_obj != NULL && !IS_ERR(slv_obj))
lu_object_put(env, &slv_obj->do_lu);
if ((req_is_acq(qb_flags) || req_is_preacq(qb_flags)) &&
OBD_FAIL_CHECK(OBD_FAIL_QUOTA_EDQUOT)) {
/* introduce inconsistency between granted value in slave index
* and slave index copy of slave */
repbody->qb_count = 0;
rc = -EDQUOT;
}
RETURN(rc);
}