/*
* Consult current disk space consumed by a given identifier.
*
* \param env - the environment passed by the caller
* \param qqi - is the pointer to the qsd_qtype_info structure associated
* with the identifier.
* \param lqe - is the quota entry associated with the identifier
*/
int qsd_refresh_usage(const struct lu_env *env, struct lquota_entry *lqe)
{
struct qsd_thread_info *qti = qsd_info(env);
struct lquota_acct_rec *rec = &qti->qti_acct_rec;
struct qsd_qtype_info *qqi = lqe2qqi(lqe);
int rc = 0;
ENTRY;
LASSERT(qqi->qqi_acct_obj);
/* read disk usage */
rc = lquota_disk_read(env, qqi->qqi_acct_obj, &lqe->lqe_id,
(struct dt_rec *)rec);
switch(rc) {
case -ENOENT:
lqe->lqe_usage = 0;
rc = 0;
break;
case 0:
if (qqi->qqi_qsd->qsd_is_md)
lqe->lqe_usage = rec->ispace;
else
lqe->lqe_usage = toqb(rec->bspace);
break;
default:
LQUOTA_ERROR(lqe, "failed to read disk usage, rc:%d", rc);
RETURN(rc);
}
LQUOTA_DEBUG(lqe, "disk usage: "LPU64, lqe->lqe_usage);
RETURN(0);
}
/*
* Update in-memory lquota entry with new quota setting from record \rec.
* The record can either be a global record (i.e. lquota_glb_rec) or a slave
* index record (i.e. lquota_slv_rec). In the former case, \global should be
* set to true.
*
* \param env - the environment passed by the caller
* \param lqe - is the quota entry associated with the identifier
* \param global - is set to true when updating the record is of type
* lquota_glb_rec. Otherwise, it is a lquota_slv_rec record.
* \param rec - is the updated record received from the master.
*/
int qsd_update_lqe(const struct lu_env *env, struct lquota_entry *lqe,
bool global, void *rec)
{
struct qsd_qtype_info *qqi;
ENTRY;
LASSERT(lqe != NULL);
LASSERT(!lqe_is_master(lqe));
qqi = lqe2qqi(lqe);
/* updating lqe is always serialized, no locking needed. */
if (global) {
struct lquota_glb_rec *glb_rec = (struct lquota_glb_rec *)rec;
/* change enforcement status based on new hard/soft limit */
lqe->lqe_enforced = (glb_rec->qbr_hardlimit ||
glb_rec->qbr_softlimit) ? true : false;
LQUOTA_DEBUG(lqe, "updating global index hardlimit: "LPU64", "
"softlimit: "LPU64, glb_rec->qbr_hardlimit,
glb_rec->qbr_softlimit);
} else {
struct lquota_slv_rec *slv_rec = (struct lquota_slv_rec *)rec;
lqe->lqe_granted = slv_rec->qsr_granted;
LQUOTA_DEBUG(lqe, "updating slave index, granted:"LPU64"",
slv_rec->qsr_granted);
}
RETURN(0);
}
/**
* Check whether a qsd instance is all set to send quota request to master.
* This includes checking whether:
* - the connection to master is set up and usable,
* - the qsd isn't stopping
* - reintegration has been successfully completed and all indexes are
* up-to-date
*
* \param lqe - is the lquota entry for which we would like to send an quota
* request
* \param lockh - is the remote handle of the global lock returned on success
*
* \retval 0 on success, appropriate error on failure
*/
static int qsd_ready(struct lquota_entry *lqe, struct lustre_handle *lockh)
{
struct qsd_qtype_info *qqi = lqe2qqi(lqe);
struct qsd_instance *qsd = qqi->qqi_qsd;
struct obd_import *imp = NULL;
struct ldlm_lock *lock;
ENTRY;
read_lock(&qsd->qsd_lock);
/* is the qsd about to shut down? */
if (qsd->qsd_stopping) {
read_unlock(&qsd->qsd_lock);
LQUOTA_DEBUG(lqe, "dropping quota req since qsd is stopping");
/* Target is about to shut down, client will retry */
RETURN(-EINPROGRESS);
}
/* is the connection to the quota master ready? */
if (qsd->qsd_exp_valid)
imp = class_exp2cliimp(qsd->qsd_exp);
if (imp == NULL || imp->imp_invalid) {
read_unlock(&qsd->qsd_lock);
LQUOTA_DEBUG(lqe, "connection to master not ready");
RETURN(-ENOTCONN);
}
/* In most case, reintegration must have been triggered (when enable
* quota or on OST start), however, in rare race condition (enabling
* quota when starting OSTs), we might miss triggering reintegration
* for some qqi.
*
* If the previous reintegration failed for some reason, we'll
* re-trigger it here as well. */
if (!qqi->qqi_glb_uptodate || !qqi->qqi_slv_uptodate) {
read_unlock(&qsd->qsd_lock);
LQUOTA_DEBUG(lqe, "not up-to-date, dropping request and "
"kicking off reintegration");
qsd_start_reint_thread(qqi);
RETURN(-EINPROGRESS);
}
/* Fill the remote global lock handle, master will check this handle
* to see if the slave is sending request with stale lock */
lustre_handle_copy(lockh, &qqi->qqi_lockh);
read_unlock(&qsd->qsd_lock);
if (!lustre_handle_is_used(lockh))
RETURN(-ENOLCK);
lock = ldlm_handle2lock(lockh);
if (lock == NULL)
RETURN(-ENOLCK);
/* return remote lock handle to be packed in quota request */
lustre_handle_copy(lockh, &lock->l_remote_handle);
LDLM_LOCK_PUT(lock);
RETURN(0);
}
/**
* Acquire \a space of quota space in order to complete an operation.
* Try to consume local quota space first and send acquire request to quota
* master if required.
*
* \param env - the environment passed by the caller
* \param lqe - is the qid entry to be processed
* \param space - is the amount of quota required for the operation
* \param ret - is the return code (-EDQUOT, -EINPROGRESS, ...)
*
* \retval true - exit from l_wait_event and real return value in \a ret
* \retval false - continue waiting
*/
static bool qsd_acquire(const struct lu_env *env, struct lquota_entry *lqe,
long long space, int *ret)
{
int rc = 0, count;
ENTRY;
for (count = 0; rc == 0; count++) {
LQUOTA_DEBUG(lqe, "acquiring:"LPD64 " count=%d", space, count);
if (lqe2qqi(lqe)->qqi_qsd->qsd_stopping) {
rc = -EINPROGRESS;
break;
}
/* refresh disk usage */
rc = qsd_refresh_usage(env, lqe);
if (rc)
break;
/* try to consume local quota space first */
rc = qsd_acquire_local(lqe, space);
if (rc != -EAGAIN)
/* rc == 0, Wouhou! enough local quota space
* rc < 0, something bad happened */
break;
/* need to acquire more quota space from master */
rc = qsd_acquire_remote(env, lqe);
}
if (rc == -EBUSY)
/* already a request in flight, continue waiting */
RETURN(false);
*ret = rc;
RETURN(true); /* exit from l_wait_event */
}
/*
* Glimpse callback handler for per-ID quota locks.
*
* \param lock - is the lock targeted by the glimpse
* \param data - is a pointer to the glimpse ptlrpc request
*/
static int qsd_id_glimpse_ast(struct ldlm_lock *lock, void *data)
{
struct ptlrpc_request *req = data;
struct lquota_entry *lqe;
struct qsd_instance *qsd;
struct ldlm_gl_lquota_desc *desc;
struct lquota_lvb *lvb;
int rc;
bool wakeup = false;
ENTRY;
rc = qsd_common_glimpse_ast(req, &desc, (void **)&lvb);
if (rc)
GOTO(out, rc);
lqe = qsd_id_ast_data_get(lock, false);
if (lqe == NULL)
/* valid race */
GOTO(out, rc = -ELDLM_NO_LOCK_DATA);
LQUOTA_DEBUG(lqe, "glimpse on quota locks, new qunit:"LPU64,
desc->gl_qunit);
qsd = lqe2qqi(lqe)->qqi_qsd;
lqe_write_lock(lqe);
lvb->lvb_id_rel = 0;
if (desc->gl_qunit != 0 && desc->gl_qunit != lqe->lqe_qunit) {
long long space;
/* extract new qunit from glimpse request */
qsd_set_qunit(lqe, desc->gl_qunit);
space = lqe->lqe_granted - lqe->lqe_pending_rel;
space -= lqe->lqe_usage;
space -= lqe->lqe_pending_write + lqe->lqe_waiting_write;
space -= lqe->lqe_qunit;
if (space > 0) {
if (lqe->lqe_pending_req > 0) {
LQUOTA_DEBUG(lqe, "request in flight, postpone "
"release of "LPD64, space);
lvb->lvb_id_may_rel = space;
} else {
lqe->lqe_pending_req++;
/* release quota space in glimpse reply */
LQUOTA_DEBUG(lqe, "releasing "LPD64, space);
lqe->lqe_granted -= space;
lvb->lvb_id_rel = space;
lqe_write_unlock(lqe);
/* change the lqe_granted */
qsd_upd_schedule(lqe2qqi(lqe), lqe, &lqe->lqe_id,
(union lquota_rec *)&lqe->lqe_granted,
0, false);
lqe_write_lock(lqe);
lqe->lqe_pending_req--;
wakeup = true;
}
}
}
lqe->lqe_edquot = !!(desc->gl_flags & LQUOTA_FL_EDQUOT);
lqe_write_unlock(lqe);
if (wakeup)
wake_up_all(&lqe->lqe_waiters);
lqe_putref(lqe);
out:
req->rq_status = rc;
RETURN(rc);
}
/**
* Adjust quota space (by acquiring or releasing) hold by the quota slave.
* This function is called after each quota request completion and during
* reintegration in order to report usage or re-acquire quota locks.
* Space adjustment is aborted if there is already a quota request in flight
* for this ID.
*
* \param env - the environment passed by the caller
* \param lqe - is the qid entry to be processed
*
* \retval 0 on success, appropriate errors on failure
*/
int qsd_adjust(const struct lu_env *env, struct lquota_entry *lqe)
{
struct qsd_thread_info *qti = qsd_info(env);
struct quota_body *qbody = &qti->qti_body;
struct qsd_instance *qsd;
struct qsd_qtype_info *qqi;
int rc;
bool intent = false;
ENTRY;
memset(qbody, 0, sizeof(*qbody));
rc = qsd_ready(lqe, &qbody->qb_glb_lockh);
if (rc) {
/* add to adjust list again to trigger adjustment later when
* slave is ready */
LQUOTA_DEBUG(lqe, "delaying adjustment since qsd isn't ready");
qsd_adjust_schedule(lqe, true, false);
RETURN(0);
}
qqi = lqe2qqi(lqe);
qsd = qqi->qqi_qsd;
lqe_write_lock(lqe);
/* fill qb_count & qb_flags */
if (!qsd_calc_adjust(lqe, qbody)) {
lqe_write_unlock(lqe);
LQUOTA_DEBUG(lqe, "no adjustment required");
RETURN(0);
}
/* only 1 quota request in flight for a given ID is allowed */
rc = qsd_request_enter(lqe);
if (rc) {
/* already a request in flight, space adjustment will be run
* again on request completion */
lqe_write_unlock(lqe);
RETURN(0);
}
if (req_is_rel(qbody->qb_flags))
lqe->lqe_pending_rel = qbody->qb_count;
lustre_handle_copy(&qti->qti_lockh, &lqe->lqe_lockh);
lqe_write_unlock(lqe);
/* hold a refcount until completion */
lqe_getref(lqe);
/* fill other quota body fields */
qbody->qb_fid = qqi->qqi_fid;
qbody->qb_id = lqe->lqe_id;
if (req_is_acq(qbody->qb_flags) || req_is_preacq(qbody->qb_flags)) {
/* check whether we own a valid lock for this ID */
rc = qsd_id_lock_match(&qti->qti_lockh, &qbody->qb_lockh);
if (rc) {
memset(&qti->qti_lockh, 0, sizeof(qti->qti_lockh));
if (req_is_preacq(qbody->qb_flags)) {
if (req_has_rep(qbody->qb_flags))
/* still want to report usage */
qbody->qb_flags = QUOTA_DQACQ_FL_REPORT;
else
/* no pre-acquire if no per-ID lock */
GOTO(out, rc = -ENOLCK);
} else {
/* no lock found, should use intent */
intent = true;
}
} else if (req_is_acq(qbody->qb_flags) &&
qbody->qb_count == 0) {
/* found cached lock, no need to acquire */
GOTO(out, rc = 0);
}
} else {
/* release and report don't need a per-ID lock */
memset(&qti->qti_lockh, 0, sizeof(qti->qti_lockh));
}
if (!intent) {
rc = qsd_send_dqacq(env, qsd->qsd_exp, qbody, false,
qsd_req_completion, qqi, &qti->qti_lockh,
lqe);
} else {
struct lquota_lvb *lvb;
OBD_ALLOC_PTR(lvb);
if (lvb == NULL)
GOTO(out, rc = -ENOMEM);
rc = qsd_intent_lock(env, qsd->qsd_exp, qbody, false,
IT_QUOTA_DQACQ, qsd_req_completion,
qqi, lvb, (void *)lqe);
}
/* the completion function will be called by qsd_send_dqacq or
* qsd_intent_lock */
RETURN(rc);
out:
qsd_req_completion(env, qqi, qbody, NULL, &qti->qti_lockh, NULL, lqe,
rc);
return rc;
}
/**
* Acquire quota space from master.
* There are at most 1 in-flight dqacq/dqrel.
*
* \param env - the environment passed by the caller
* \param lqe - is the qid entry to be processed
*
* \retval 0 - success
* \retval -EDQUOT - out of quota
* \retval -EINPROGRESS - inform client to retry write/create
* \retval -EBUSY - already a quota request in flight
* \retval -ve - other appropriate errors
*/
static int qsd_acquire_remote(const struct lu_env *env,
struct lquota_entry *lqe)
{
struct qsd_thread_info *qti = qsd_info(env);
struct quota_body *qbody = &qti->qti_body;
struct qsd_instance *qsd;
struct qsd_qtype_info *qqi;
int rc;
ENTRY;
memset(qbody, 0, sizeof(*qbody));
rc = qsd_ready(lqe, &qbody->qb_glb_lockh);
if (rc)
RETURN(rc);
qqi = lqe2qqi(lqe);
qsd = qqi->qqi_qsd;
lqe_write_lock(lqe);
/* is quota really enforced for this id? */
if (!lqe->lqe_enforced) {
lqe_write_unlock(lqe);
LQUOTA_DEBUG(lqe, "quota not enforced any more");
RETURN(0);
}
/* fill qb_count & qb_flags */
if (!qsd_calc_acquire(lqe, qbody)) {
lqe_write_unlock(lqe);
LQUOTA_DEBUG(lqe, "No acquire required");
RETURN(0);
}
/* check whether an acquire request completed recently */
if (lqe->lqe_acq_rc != 0 &&
cfs_time_before_64(cfs_time_shift_64(-1), lqe->lqe_acq_time)) {
lqe_write_unlock(lqe);
LQUOTA_DEBUG(lqe, "using cached return code %d", lqe->lqe_acq_rc);
RETURN(lqe->lqe_acq_rc);
}
/* only 1 quota request in flight for a given ID is allowed */
rc = qsd_request_enter(lqe);
if (rc) {
lqe_write_unlock(lqe);
RETURN(rc);
}
lustre_handle_copy(&qti->qti_lockh, &lqe->lqe_lockh);
lqe_write_unlock(lqe);
/* hold a refcount until completion */
lqe_getref(lqe);
/* fill other quota body fields */
qbody->qb_fid = qqi->qqi_fid;
qbody->qb_id = lqe->lqe_id;
/* check whether we already own a valid lock for this ID */
rc = qsd_id_lock_match(&qti->qti_lockh, &qbody->qb_lockh);
if (rc) {
struct lquota_lvb *lvb;
OBD_ALLOC_PTR(lvb);
if (lvb == NULL) {
rc = -ENOMEM;
qsd_req_completion(env, qqi, qbody, NULL,
&qti->qti_lockh, NULL, lqe, rc);
RETURN(rc);
}
/* no lock found, should use intent */
rc = qsd_intent_lock(env, qsd->qsd_exp, qbody, true,
IT_QUOTA_DQACQ, qsd_req_completion,
qqi, lvb, (void *)lqe);
} else {
/* lock found, should use regular dqacq */
rc = qsd_send_dqacq(env, qsd->qsd_exp, qbody, true,
qsd_req_completion, qqi, &qti->qti_lockh,
lqe);
}
/* the completion function will be called by qsd_send_dqacq or
* qsd_intent_lock */
RETURN(rc);
}
