/*
* Blocking callback handler for global index lock
*
* \param lock - is the lock for which ast occurred.
* \param desc - is the description of a conflicting lock in case of blocking
* ast.
* \param data - is the value of lock->l_ast_data
* \param flag - LDLM_CB_BLOCKING or LDLM_CB_CANCELING. Used to distinguish
* cancellation and blocking ast's.
*/
static int qsd_glb_blocking_ast(struct ldlm_lock *lock,
struct ldlm_lock_desc *desc, void *data,
int flag)
{
int rc = 0;
ENTRY;
switch(flag) {
case LDLM_CB_BLOCKING: {
struct lustre_handle lockh;
LDLM_DEBUG(lock, "blocking AST on global quota lock");
ldlm_lock2handle(lock, &lockh);
rc = ldlm_cli_cancel(&lockh, LCF_ASYNC);
break;
}
case LDLM_CB_CANCELING: {
struct qsd_qtype_info *qqi;
LDLM_DEBUG(lock, "canceling global quota lock");
qqi = qsd_glb_ast_data_get(lock, true);
if (qqi == NULL)
break;
/* we are losing the global index lock, so let's mark the
* global & slave indexes as not up-to-date any more */
write_lock(&qqi->qqi_qsd->qsd_lock);
qqi->qqi_glb_uptodate = false;
qqi->qqi_slv_uptodate = false;
if (lock->l_handle.h_cookie == qqi->qqi_lockh.cookie)
memset(&qqi->qqi_lockh, 0, sizeof(qqi->qqi_lockh));
write_unlock(&qqi->qqi_qsd->qsd_lock);
CDEBUG(D_QUOTA, "%s: losing global index lock for %s type\n",
qqi->qqi_qsd->qsd_svname, QTYPE_NAME((qqi->qqi_qtype)));
/* kick off reintegration thread if not running already, if
* it's just local cancel (for stack clean up or eviction),
* don't re-trigger the reintegration. */
if (!ldlm_is_local_only(lock))
qsd_start_reint_thread(qqi);
lu_ref_del(&qqi->qqi_reference, "ast_data_get", lock);
qqi_putref(qqi);
break;
}
default:
LASSERTF(0, "invalid flags for blocking ast %d", flag);
}
RETURN(rc);
}
/*
* Print lqe information for debugging.
*
* \param lqe - is the quota entry to debug
* \param arg - is the pointer to the qsd_qtype_info structure
* \param msgdata - debug message
* \param fmt - format of debug message
*/
static void qsd_lqe_debug(struct lquota_entry *lqe, void *arg,
struct libcfs_debug_msg_data *msgdata,
const char *fmt, va_list args)
{
struct qsd_qtype_info *qqi = (struct qsd_qtype_info *)arg;
libcfs_debug_vmsg2(msgdata, fmt, args,
"qsd:%s qtype:%s id:"LPU64" enforced:%d granted:"
LPU64" pending:"LPU64" waiting:"LPU64" req:%d usage:"
LPU64" qunit:"LPU64" qtune:"LPU64" edquot:%d\n",
qqi->qqi_qsd->qsd_svname, QTYPE_NAME(qqi->qqi_qtype),
lqe->lqe_id.qid_uid, lqe->lqe_enforced,
lqe->lqe_granted, lqe->lqe_pending_write,
lqe->lqe_waiting_write, lqe->lqe_pending_req,
lqe->lqe_usage, lqe->lqe_qunit, lqe->lqe_qtune,
lqe->lqe_edquot);
}
/**
* Initialize slave index object to collect local quota limit for user or group.
*
* \param env - is the environment passed by the caller
* \param dev - is the dt_device storing the slave index object
* \param type - is the quota type, either USRQUOTA or GRPQUOTA
*/
static struct dt_object *quota_obj_lookup(const struct lu_env *env,
struct dt_device *dev, int type)
{
struct lquota_thread_info *qti = lquota_info(env);
struct dt_object *obj = NULL;
ENTRY;
qti->qti_fid.f_seq = FID_SEQ_QUOTA;
qti->qti_fid.f_oid = type == USRQUOTA ? LQUOTA_USR_OID : LQUOTA_GRP_OID;
qti->qti_fid.f_ver = 0;
/* lookup the quota object */
obj = dt_locate(env, dev, &qti->qti_fid);
if (IS_ERR(obj))
RETURN(obj);
if (!dt_object_exists(obj)) {
lu_object_put(env, &obj->do_lu);
RETURN(ERR_PTR(-ENOENT));
}
if (obj->do_index_ops == NULL) {
int rc;
/* set up indexing operations */
rc = obj->do_ops->do_index_try(env, obj,
&dt_quota_slv_features);
if (rc) {
CERROR("%s: failed to set up indexing operations for %s"
" slave index object rc:%d\n",
dev->dd_lu_dev.ld_obd->obd_name,
QTYPE_NAME(type), rc);
lu_object_put(env, &obj->do_lu);
RETURN(ERR_PTR(rc));
}
}
RETURN(obj);
}
/**
* Look-up accounting object to collect space usage information for user
* or group.
*
* \param env - is the environment passed by the caller
* \param dev - is the dt_device storing the accounting object
* \param type - is the quota type, either USRQUOTA or GRPQUOTA
*/
struct dt_object *acct_obj_lookup(const struct lu_env *env,
struct dt_device *dev, int type)
{
struct lquota_thread_info *qti = lquota_info(env);
struct dt_object *obj = NULL;
ENTRY;
lu_local_obj_fid(&qti->qti_fid,
type == USRQUOTA ? ACCT_USER_OID : ACCT_GROUP_OID);
/* lookup the accounting object */
obj = dt_locate(env, dev, &qti->qti_fid);
if (IS_ERR(obj))
RETURN(obj);
if (!dt_object_exists(obj)) {
lu_object_put(env, &obj->do_lu);
RETURN(ERR_PTR(-ENOENT));
}
if (obj->do_index_ops == NULL) {
int rc;
/* set up indexing operations */
rc = obj->do_ops->do_index_try(env, obj, &dt_acct_features);
if (rc) {
CERROR("%s: failed to set up indexing operations for %s"
" acct object rc:%d\n",
dev->dd_lu_dev.ld_obd->obd_name,
QTYPE_NAME(type), rc);
lu_object_put(env, &obj->do_lu);
RETURN(ERR_PTR(rc));
}
}
RETURN(obj);
}
/*
* Initialize on-disk structures in order to manage quota enforcement for
* the target associated with the qsd instance \qsd and starts the reintegration
* procedure for each quota type as soon as possible.
* The last step of the reintegration will be completed once qsd_start() is
* called, at which points the space reconciliation with the master will be
* executed.
* This function must be called when the server stack is fully configured,
* typically when ->ldo_prepare is called across the stack.
*
* \param env - the environment passed by the caller
* \param qsd - is qsd_instance to prepare
*
* \retval - 0 on success, appropriate error on failure
*/
int qsd_prepare(const struct lu_env *env, struct qsd_instance *qsd)
{
struct qsd_thread_info *qti = qsd_info(env);
int qtype, rc = 0;
ENTRY;
if (unlikely(qsd == NULL))
RETURN(0);
read_lock(&qsd->qsd_lock);
if (qsd->qsd_prepared) {
CERROR("%s: qsd instance already prepared\n", qsd->qsd_svname);
rc = -EALREADY;
}
read_unlock(&qsd->qsd_lock);
if (rc)
RETURN(rc);
/* Record whether this qsd instance is managing quota enforcement for a
* MDT (i.e. inode quota) or OST (block quota) */
if (lu_device_is_md(qsd->qsd_dev->dd_lu_dev.ld_site->ls_top_dev)) {
qsd->qsd_is_md = true;
qsd->qsd_sync_threshold = LQUOTA_LEAST_QUNIT(LQUOTA_RES_MD);
} else {
qsd->qsd_sync_threshold = LQUOTA_LEAST_QUNIT(LQUOTA_RES_DT);
}
/* look-up on-disk directory for the quota slave */
qsd->qsd_root = lquota_disk_dir_find_create(env, qsd->qsd_dev, NULL,
QSD_DIR);
if (IS_ERR(qsd->qsd_root)) {
rc = PTR_ERR(qsd->qsd_root);
qsd->qsd_root = NULL;
CERROR("%s: failed to create quota slave root dir (%d)\n",
qsd->qsd_svname, rc);
RETURN(rc);
}
/* initialize per-quota type data */
for (qtype = USRQUOTA; qtype < MAXQUOTAS; qtype++) {
rc = qsd_qtype_init(env, qsd, qtype);
if (rc)
RETURN(rc);
}
/* pools successfully setup, mark the qsd as prepared */
write_lock(&qsd->qsd_lock);
qsd->qsd_prepared = true;
write_unlock(&qsd->qsd_lock);
/* start reintegration thread for each type, if required */
for (qtype = USRQUOTA; qtype < MAXQUOTAS; qtype++) {
struct qsd_qtype_info *qqi = qsd->qsd_type_array[qtype];
if (qsd_type_enabled(qsd, qtype) && qsd->qsd_acct_failed) {
LCONSOLE_ERROR("%s: can't enable quota enforcement "
"since space accounting isn't functional"
". Please run tunefs.lustre --quota on "
"an unmounted filesystem if not done "
"already\n", qsd->qsd_svname);
break;
}
rc = qsd_start_reint_thread(qqi);
if (rc) {
CERROR("%s: failed to start reint thread for type %s "
"(%d)\n", qsd->qsd_svname, QTYPE_NAME(qtype),
rc);
RETURN(rc);
}
}
/* start writeback thread */
rc = qsd_start_upd_thread(qsd);
if (rc) {
CERROR("%s: failed to start writeback thread (%d)\n",
qsd->qsd_svname, rc);
RETURN(rc);
}
/* generate osp name */
rc = tgt_name2lwp_name(qsd->qsd_svname, qti->qti_buf,
MTI_NAME_MAXLEN, 0);
if (rc) {
CERROR("%s: failed to generate ospname (%d)\n",
qsd->qsd_svname, rc);
RETURN(rc);
}
/* the connection callback will start the reintegration
* procedure if quota is enabled */
rc = lustre_register_lwp_item(qti->qti_buf, &qsd->qsd_exp,
qsd_conn_callback, (void *)qsd);
if (rc) {
CERROR("%s: fail to get connection to master (%d)\n",
qsd->qsd_svname, rc);
RETURN(rc);
}
RETURN(0);
}
/*
* Allocate and initialize a qsd_qtype_info structure for quota type \qtype.
* This opens the accounting object and initializes the proc file.
* It's called on OSD start when the qsd_prepare() is invoked on the qsd
* instance.
*
* \param env - the environment passed by the caller
* \param qsd - is the qsd instance which will be in charge of the new
* qsd_qtype_info instance.
* \param qtype - is quota type to set up
*
* \retval - 0 on success and qsd->qsd_type_array[qtype] is allocated,
* appropriate error on failure
*/
static int qsd_qtype_init(const struct lu_env *env, struct qsd_instance *qsd,
int qtype)
{
struct qsd_qtype_info *qqi;
int rc;
struct obd_uuid uuid;
ENTRY;
LASSERT(qsd->qsd_type_array[qtype] == NULL);
/* allocate structure for this quota type */
OBD_ALLOC_PTR(qqi);
if (qqi == NULL)
RETURN(-ENOMEM);
qsd->qsd_type_array[qtype] = qqi;
atomic_set(&qqi->qqi_ref, 1); /* referenced from qsd */
/* set backpointer and other parameters */
qqi->qqi_qsd = qsd;
qqi->qqi_qtype = qtype;
lu_ref_init(&qqi->qqi_reference);
lquota_generate_fid(&qqi->qqi_fid, qsd->qsd_pool_id, QSD_RES_TYPE(qsd),
qtype);
qqi->qqi_glb_uptodate = false;
qqi->qqi_slv_uptodate = false;
qqi->qqi_reint = false;
init_waitqueue_head(&qqi->qqi_reint_thread.t_ctl_waitq);
thread_set_flags(&qqi->qqi_reint_thread, SVC_STOPPED);
CFS_INIT_LIST_HEAD(&qqi->qqi_deferred_glb);
CFS_INIT_LIST_HEAD(&qqi->qqi_deferred_slv);
/* open accounting object */
LASSERT(qqi->qqi_acct_obj == NULL);
qqi->qqi_acct_obj = acct_obj_lookup(env, qsd->qsd_dev, qtype);
if (IS_ERR(qqi->qqi_acct_obj)) {
CDEBUG(D_QUOTA, "%s: no %s space accounting support rc:%ld\n",
qsd->qsd_svname, QTYPE_NAME(qtype),
PTR_ERR(qqi->qqi_acct_obj));
qqi->qqi_acct_obj = NULL;
qsd->qsd_acct_failed = true;
}
/* open global index copy */
LASSERT(qqi->qqi_glb_obj == NULL);
qqi->qqi_glb_obj = lquota_disk_glb_find_create(env, qsd->qsd_dev,
qsd->qsd_root,
&qqi->qqi_fid, true);
if (IS_ERR(qqi->qqi_glb_obj)) {
CERROR("%s: can't open global index copy "DFID" %ld\n",
qsd->qsd_svname, PFID(&qqi->qqi_fid),
PTR_ERR(qqi->qqi_glb_obj));
GOTO(out, rc = PTR_ERR(qqi->qqi_glb_obj));
}
qqi->qqi_glb_ver = dt_version_get(env, qqi->qqi_glb_obj);
/* open slave index copy */
LASSERT(qqi->qqi_slv_obj == NULL);
obd_str2uuid(&uuid, qsd->qsd_svname);
qqi->qqi_slv_obj = lquota_disk_slv_find_create(env, qsd->qsd_dev,
qsd->qsd_root,
&qqi->qqi_fid, &uuid,
true);
if (IS_ERR(qqi->qqi_slv_obj)) {
CERROR("%s: can't open slave index copy "DFID" %ld\n",
qsd->qsd_svname, PFID(&qqi->qqi_fid),
PTR_ERR(qqi->qqi_slv_obj));
GOTO(out, rc = PTR_ERR(qqi->qqi_slv_obj));
}
qqi->qqi_slv_ver = dt_version_get(env, qqi->qqi_slv_obj);
/* allocate site */
qqi->qqi_site = lquota_site_alloc(env, qqi, false, qtype, &qsd_lqe_ops);
if (IS_ERR(qqi->qqi_site)) {
CERROR("%s: can't allocate site "DFID" %ld\n", qsd->qsd_svname,
PFID(&qqi->qqi_fid), PTR_ERR(qqi->qqi_site));
GOTO(out, rc = PTR_ERR(qqi->qqi_site));
}
/* register proc entry for accounting & global index copy objects */
rc = lprocfs_seq_create(qsd->qsd_proc,
qtype == USRQUOTA ? "acct_user" : "acct_group",
0444, &lprocfs_quota_seq_fops,
qqi->qqi_acct_obj);
if (rc) {
CERROR("%s: can't add procfs entry for accounting file %d\n",
qsd->qsd_svname, rc);
GOTO(out, rc);
}
rc = lprocfs_seq_create(qsd->qsd_proc,
qtype == USRQUOTA ? "limit_user" : "limit_group",
0444, &lprocfs_quota_seq_fops,
qqi->qqi_glb_obj);
if (rc) {
CERROR("%s: can't add procfs entry for global index copy %d\n",
qsd->qsd_svname, rc);
GOTO(out, rc);
}
EXIT;
out:
if (rc)
qsd_qtype_fini(env, qsd, qtype);
return rc;
}
