/**
* Check whether a qsd instance is all set to send quota request to master.
* This includes checking whether:
* - the connection to master is set up and usable,
* - the qsd isn't stopping
* - reintegration has been successfully completed and all indexes are
* up-to-date
*
* \param lqe - is the lquota entry for which we would like to send an quota
* request
* \param lockh - is the remote handle of the global lock returned on success
*
* \retval 0 on success, appropriate error on failure
*/
static int qsd_ready(struct lquota_entry *lqe, struct lustre_handle *lockh)
{
struct qsd_qtype_info *qqi = lqe2qqi(lqe);
struct qsd_instance *qsd = qqi->qqi_qsd;
struct obd_import *imp = NULL;
struct ldlm_lock *lock;
ENTRY;
read_lock(&qsd->qsd_lock);
/* is the qsd about to shut down? */
if (qsd->qsd_stopping) {
read_unlock(&qsd->qsd_lock);
LQUOTA_DEBUG(lqe, "dropping quota req since qsd is stopping");
/* Target is about to shut down, client will retry */
RETURN(-EINPROGRESS);
}
/* is the connection to the quota master ready? */
if (qsd->qsd_exp_valid)
imp = class_exp2cliimp(qsd->qsd_exp);
if (imp == NULL || imp->imp_invalid) {
read_unlock(&qsd->qsd_lock);
LQUOTA_DEBUG(lqe, "connection to master not ready");
RETURN(-ENOTCONN);
}
/* In most case, reintegration must have been triggered (when enable
* quota or on OST start), however, in rare race condition (enabling
* quota when starting OSTs), we might miss triggering reintegration
* for some qqi.
*
* If the previous reintegration failed for some reason, we'll
* re-trigger it here as well. */
if (!qqi->qqi_glb_uptodate || !qqi->qqi_slv_uptodate) {
read_unlock(&qsd->qsd_lock);
LQUOTA_DEBUG(lqe, "not up-to-date, dropping request and "
"kicking off reintegration");
qsd_start_reint_thread(qqi);
RETURN(-EINPROGRESS);
}
/* Fill the remote global lock handle, master will check this handle
* to see if the slave is sending request with stale lock */
lustre_handle_copy(lockh, &qqi->qqi_lockh);
read_unlock(&qsd->qsd_lock);
if (!lustre_handle_is_used(lockh))
RETURN(-ENOLCK);
lock = ldlm_handle2lock(lockh);
if (lock == NULL)
RETURN(-ENOLCK);
/* return remote lock handle to be packed in quota request */
lustre_handle_copy(lockh, &lock->l_remote_handle);
LDLM_LOCK_PUT(lock);
RETURN(0);
}
/*
* Blocking callback handler for global index lock
*
* \param lock - is the lock for which ast occurred.
* \param desc - is the description of a conflicting lock in case of blocking
* ast.
* \param data - is the value of lock->l_ast_data
* \param flag - LDLM_CB_BLOCKING or LDLM_CB_CANCELING. Used to distinguish
* cancellation and blocking ast's.
*/
static int qsd_glb_blocking_ast(struct ldlm_lock *lock,
struct ldlm_lock_desc *desc, void *data,
int flag)
{
int rc = 0;
ENTRY;
switch(flag) {
case LDLM_CB_BLOCKING: {
struct lustre_handle lockh;
LDLM_DEBUG(lock, "blocking AST on global quota lock");
ldlm_lock2handle(lock, &lockh);
rc = ldlm_cli_cancel(&lockh, LCF_ASYNC);
break;
}
case LDLM_CB_CANCELING: {
struct qsd_qtype_info *qqi;
LDLM_DEBUG(lock, "canceling global quota lock");
qqi = qsd_glb_ast_data_get(lock, true);
if (qqi == NULL)
break;
/* we are losing the global index lock, so let's mark the
* global & slave indexes as not up-to-date any more */
write_lock(&qqi->qqi_qsd->qsd_lock);
qqi->qqi_glb_uptodate = false;
qqi->qqi_slv_uptodate = false;
if (lock->l_handle.h_cookie == qqi->qqi_lockh.cookie)
memset(&qqi->qqi_lockh, 0, sizeof(qqi->qqi_lockh));
write_unlock(&qqi->qqi_qsd->qsd_lock);
CDEBUG(D_QUOTA, "%s: losing global index lock for %s type\n",
qqi->qqi_qsd->qsd_svname, QTYPE_NAME((qqi->qqi_qtype)));
/* kick off reintegration thread if not running already, if
* it's just local cancel (for stack clean up or eviction),
* don't re-trigger the reintegration. */
if (!ldlm_is_local_only(lock))
qsd_start_reint_thread(qqi);
lu_ref_del(&qqi->qqi_reference, "ast_data_get", lock);
qqi_putref(qqi);
break;
}
default:
LASSERTF(0, "invalid flags for blocking ast %d", flag);
}
RETURN(rc);
}
/* force reintegration procedure to be executed.
* Used for test/debugging purpose */
static ssize_t
lprocfs_force_reint_seq_write(struct file *file, const char *buffer,
size_t count, loff_t *off)
{
struct qsd_instance *qsd = ((struct seq_file *)file->private_data)->private;
int rc = 0, qtype;
LASSERT(qsd != NULL);
write_lock(&qsd->qsd_lock);
if (qsd->qsd_stopping) {
/* don't mess up with shutdown procedure, it is already
* complicated enough */
rc = -ESHUTDOWN;
} else if (!qsd->qsd_prepared) {
rc = -EAGAIN;
} else {
/* mark all indexes as stale */
for (qtype = USRQUOTA; qtype < MAXQUOTAS; qtype++) {
qsd->qsd_type_array[qtype]->qqi_glb_uptodate = false;
qsd->qsd_type_array[qtype]->qqi_slv_uptodate = false;
}
}
write_unlock(&qsd->qsd_lock);
if (rc)
return rc;
/* kick off reintegration */
for (qtype = USRQUOTA; qtype < MAXQUOTAS; qtype++) {
rc = qsd_start_reint_thread(qsd->qsd_type_array[qtype]);
if (rc)
break;
}
return rc == 0 ? count : rc;
}
/*
* Initialize on-disk structures in order to manage quota enforcement for
* the target associated with the qsd instance \qsd and starts the reintegration
* procedure for each quota type as soon as possible.
* The last step of the reintegration will be completed once qsd_start() is
* called, at which points the space reconciliation with the master will be
* executed.
* This function must be called when the server stack is fully configured,
* typically when ->ldo_prepare is called across the stack.
*
* \param env - the environment passed by the caller
* \param qsd - is qsd_instance to prepare
*
* \retval - 0 on success, appropriate error on failure
*/
int qsd_prepare(const struct lu_env *env, struct qsd_instance *qsd)
{
struct qsd_thread_info *qti = qsd_info(env);
int qtype, rc = 0;
ENTRY;
if (unlikely(qsd == NULL))
RETURN(0);
read_lock(&qsd->qsd_lock);
if (qsd->qsd_prepared) {
CERROR("%s: qsd instance already prepared\n", qsd->qsd_svname);
rc = -EALREADY;
}
read_unlock(&qsd->qsd_lock);
if (rc)
RETURN(rc);
/* Record whether this qsd instance is managing quota enforcement for a
* MDT (i.e. inode quota) or OST (block quota) */
if (lu_device_is_md(qsd->qsd_dev->dd_lu_dev.ld_site->ls_top_dev)) {
qsd->qsd_is_md = true;
qsd->qsd_sync_threshold = LQUOTA_LEAST_QUNIT(LQUOTA_RES_MD);
} else {
qsd->qsd_sync_threshold = LQUOTA_LEAST_QUNIT(LQUOTA_RES_DT);
}
/* look-up on-disk directory for the quota slave */
qsd->qsd_root = lquota_disk_dir_find_create(env, qsd->qsd_dev, NULL,
QSD_DIR);
if (IS_ERR(qsd->qsd_root)) {
rc = PTR_ERR(qsd->qsd_root);
qsd->qsd_root = NULL;
CERROR("%s: failed to create quota slave root dir (%d)\n",
qsd->qsd_svname, rc);
RETURN(rc);
}
/* initialize per-quota type data */
for (qtype = USRQUOTA; qtype < MAXQUOTAS; qtype++) {
rc = qsd_qtype_init(env, qsd, qtype);
if (rc)
RETURN(rc);
}
/* pools successfully setup, mark the qsd as prepared */
write_lock(&qsd->qsd_lock);
qsd->qsd_prepared = true;
write_unlock(&qsd->qsd_lock);
/* start reintegration thread for each type, if required */
for (qtype = USRQUOTA; qtype < MAXQUOTAS; qtype++) {
struct qsd_qtype_info *qqi = qsd->qsd_type_array[qtype];
if (qsd_type_enabled(qsd, qtype) && qsd->qsd_acct_failed) {
LCONSOLE_ERROR("%s: can't enable quota enforcement "
"since space accounting isn't functional"
". Please run tunefs.lustre --quota on "
"an unmounted filesystem if not done "
"already\n", qsd->qsd_svname);
break;
}
rc = qsd_start_reint_thread(qqi);
if (rc) {
CERROR("%s: failed to start reint thread for type %s "
"(%d)\n", qsd->qsd_svname, QTYPE_NAME(qtype),
rc);
RETURN(rc);
}
}
/* start writeback thread */
rc = qsd_start_upd_thread(qsd);
if (rc) {
CERROR("%s: failed to start writeback thread (%d)\n",
qsd->qsd_svname, rc);
RETURN(rc);
}
/* generate osp name */
rc = tgt_name2lwp_name(qsd->qsd_svname, qti->qti_buf,
MTI_NAME_MAXLEN, 0);
if (rc) {
CERROR("%s: failed to generate ospname (%d)\n",
qsd->qsd_svname, rc);
RETURN(rc);
}
/* the connection callback will start the reintegration
* procedure if quota is enabled */
rc = lustre_register_lwp_item(qti->qti_buf, &qsd->qsd_exp,
qsd_conn_callback, (void *)qsd);
if (rc) {
CERROR("%s: fail to get connection to master (%d)\n",
qsd->qsd_svname, rc);
RETURN(rc);
}
RETURN(0);
}