static void osp_statfs_timer_cb(unsigned long _d) { struct osp_device *d = (struct osp_device *) _d; LASSERT(d); if (d->opd_pre != NULL && osp_precreate_running(d)) wake_up(&d->opd_pre_waitq); }
int osp_init_precreate(struct osp_device *d) { struct l_wait_info lwi = { 0 }; struct task_struct *task; ENTRY; OBD_ALLOC_PTR(d->opd_pre); if (d->opd_pre == NULL) RETURN(-ENOMEM); /* initially precreation isn't ready */ d->opd_pre_status = -EAGAIN; fid_zero(&d->opd_pre_used_fid); d->opd_pre_used_fid.f_oid = 1; fid_zero(&d->opd_pre_last_created_fid); d->opd_pre_last_created_fid.f_oid = 1; d->opd_pre_reserved = 0; d->opd_got_disconnected = 1; d->opd_pre_grow_slow = 0; d->opd_pre_grow_count = OST_MIN_PRECREATE; d->opd_pre_min_grow_count = OST_MIN_PRECREATE; d->opd_pre_max_grow_count = OST_MAX_PRECREATE; spin_lock_init(&d->opd_pre_lock); init_waitqueue_head(&d->opd_pre_waitq); init_waitqueue_head(&d->opd_pre_user_waitq); init_waitqueue_head(&d->opd_pre_thread.t_ctl_waitq); /* * Initialize statfs-related things */ d->opd_statfs_maxage = 5; /* default update interval */ d->opd_statfs_fresh_till = cfs_time_shift(-1000); CDEBUG(D_OTHER, "current %llu, fresh till %llu\n", (unsigned long long)cfs_time_current(), (unsigned long long)d->opd_statfs_fresh_till); cfs_timer_init(&d->opd_statfs_timer, osp_statfs_timer_cb, d); /* * start thread handling precreation and statfs updates */ task = kthread_run(osp_precreate_thread, d, "osp-pre-%u-%u", d->opd_index, d->opd_group); if (IS_ERR(task)) { CERROR("can't start precreate thread %ld\n", PTR_ERR(task)); RETURN(PTR_ERR(task)); } l_wait_event(d->opd_pre_thread.t_ctl_waitq, osp_precreate_running(d) || osp_precreate_stopped(d), &lwi); RETURN(0); }
int osp_init_precreate(struct osp_device *d) { struct l_wait_info lwi = { 0 }; int rc; ENTRY; /* initially precreation isn't ready */ d->opd_pre_status = -EAGAIN; fid_zero(&d->opd_pre_used_fid); d->opd_pre_used_fid.f_oid = 1; fid_zero(&d->opd_pre_last_created_fid); d->opd_pre_last_created_fid.f_oid = 1; d->opd_pre_reserved = 0; d->opd_got_disconnected = 1; d->opd_pre_grow_slow = 0; d->opd_pre_grow_count = OST_MIN_PRECREATE; d->opd_pre_min_grow_count = OST_MIN_PRECREATE; d->opd_pre_max_grow_count = OST_MAX_PRECREATE; spin_lock_init(&d->opd_pre_lock); cfs_waitq_init(&d->opd_pre_waitq); cfs_waitq_init(&d->opd_pre_user_waitq); cfs_waitq_init(&d->opd_pre_thread.t_ctl_waitq); /* * Initialize statfs-related things */ d->opd_statfs_maxage = 5; /* default update interval */ d->opd_statfs_fresh_till = cfs_time_shift(-1000); CDEBUG(D_OTHER, "current %llu, fresh till %llu\n", (unsigned long long)cfs_time_current(), (unsigned long long)d->opd_statfs_fresh_till); cfs_timer_init(&d->opd_statfs_timer, osp_statfs_timer_cb, d); /* * start thread handling precreation and statfs updates */ rc = cfs_create_thread(osp_precreate_thread, d, 0); if (rc < 0) { CERROR("can't start precreate thread %d\n", rc); RETURN(rc); } l_wait_event(d->opd_pre_thread.t_ctl_waitq, osp_precreate_running(d) || osp_precreate_stopped(d), &lwi); RETURN(0); }
static int osp_statfs_interpret(const struct lu_env *env, struct ptlrpc_request *req, union ptlrpc_async_args *aa, int rc) { struct obd_import *imp = req->rq_import; struct obd_statfs *msfs; struct osp_device *d; ENTRY; aa = ptlrpc_req_async_args(req); d = aa->pointer_arg[0]; LASSERT(d); if (rc != 0) GOTO(out, rc); msfs = req_capsule_server_get(&req->rq_pill, &RMF_OBD_STATFS); if (msfs == NULL) GOTO(out, rc = -EPROTO); d->opd_statfs = *msfs; osp_pre_update_status(d, rc); /* schedule next update */ d->opd_statfs_fresh_till = cfs_time_shift(d->opd_statfs_maxage); cfs_timer_arm(&d->opd_statfs_timer, d->opd_statfs_fresh_till); d->opd_statfs_update_in_progress = 0; CDEBUG(D_CACHE, "updated statfs %p\n", d); RETURN(0); out: /* couldn't update statfs, try again as soon as possible */ if (d->opd_pre != NULL && osp_precreate_running(d)) wake_up(&d->opd_pre_waitq); if (req->rq_import_generation == imp->imp_generation) CDEBUG(D_CACHE, "%s: couldn't update statfs: rc = %d\n", d->opd_obd->obd_name, rc); RETURN(rc); }
static int osp_precreate_thread(void *_arg) { struct osp_device *d = _arg; struct ptlrpc_thread *thread = &d->opd_pre_thread; struct l_wait_info lwi = { 0 }; struct lu_env env; int rc; ENTRY; rc = lu_env_init(&env, d->opd_dt_dev.dd_lu_dev.ld_type->ldt_ctx_tags); if (rc) { CERROR("%s: init env error: rc = %d\n", d->opd_obd->obd_name, rc); RETURN(rc); } spin_lock(&d->opd_pre_lock); thread->t_flags = SVC_RUNNING; spin_unlock(&d->opd_pre_lock); wake_up(&thread->t_ctl_waitq); while (osp_precreate_running(d)) { /* * need to be connected to OST */ while (osp_precreate_running(d)) { l_wait_event(d->opd_pre_waitq, !osp_precreate_running(d) || d->opd_new_connection, &lwi); if (!d->opd_new_connection) continue; d->opd_new_connection = 0; d->opd_got_disconnected = 0; break; } if (!osp_precreate_running(d)) break; LASSERT(d->opd_obd->u.cli.cl_seq != NULL); if (d->opd_obd->u.cli.cl_seq->lcs_exp == NULL) { /* Get new sequence for client first */ LASSERT(d->opd_exp != NULL); d->opd_obd->u.cli.cl_seq->lcs_exp = class_export_get(d->opd_exp); rc = osp_init_pre_fid(d); if (rc != 0) { class_export_put(d->opd_exp); d->opd_obd->u.cli.cl_seq->lcs_exp = NULL; CERROR("%s: init pre fid error: rc = %d\n", d->opd_obd->obd_name, rc); continue; } } osp_statfs_update(d); /* * Clean up orphans or recreate missing objects. */ rc = osp_precreate_cleanup_orphans(&env, d); if (rc != 0) continue; /* * connected, can handle precreates now */ while (osp_precreate_running(d)) { l_wait_event(d->opd_pre_waitq, !osp_precreate_running(d) || osp_precreate_near_empty(&env, d) || osp_statfs_need_update(d) || d->opd_got_disconnected, &lwi); if (!osp_precreate_running(d)) break; /* something happened to the connection * have to start from the beginning */ if (d->opd_got_disconnected) break; if (osp_statfs_need_update(d)) osp_statfs_update(d); /* To avoid handling different seq in precreate/orphan * cleanup, it will hold precreate until current seq is * used up. */ if (unlikely(osp_precreate_end_seq(&env, d) && !osp_create_end_seq(&env, d))) continue; if (unlikely(osp_precreate_end_seq(&env, d) && osp_create_end_seq(&env, d))) { LCONSOLE_INFO("%s:"LPX64" is used up." " Update to new seq\n", d->opd_obd->obd_name, fid_seq(&d->opd_pre_last_created_fid)); rc = osp_precreate_rollover_new_seq(&env, d); if (rc) continue; } if (osp_precreate_near_empty(&env, d)) { rc = osp_precreate_send(&env, d); /* osp_precreate_send() sets opd_pre_status * in case of error, that prevent the using of * failed device. */ if (rc < 0 && rc != -ENOSPC && rc != -ETIMEDOUT && rc != -ENOTCONN) CERROR("%s: cannot precreate objects:" " rc = %d\n", d->opd_obd->obd_name, rc); } } } thread->t_flags = SVC_STOPPED; lu_env_fini(&env); wake_up(&thread->t_ctl_waitq); RETURN(0); }
/** * asks OST to clean precreate orphans * and gets next id for new objects */ static int osp_precreate_cleanup_orphans(struct lu_env *env, struct osp_device *d) { struct osp_thread_info *osi = osp_env_info(env); struct lu_fid *last_fid = &osi->osi_fid; struct ptlrpc_request *req = NULL; struct obd_import *imp; struct ost_body *body; struct l_wait_info lwi = { 0 }; int update_status = 0; int rc; int diff; ENTRY; /* * wait for local recovery to finish, so we can cleanup orphans * orphans are all objects since "last used" (assigned), but * there might be objects reserved and in some cases they won't * be used. we can't cleanup them till we're sure they won't be * used. also can't we allow new reservations because they may * end up getting orphans being cleaned up below. so we block * new reservations and wait till all reserved objects either * user or released. */ spin_lock(&d->opd_pre_lock); d->opd_pre_recovering = 1; spin_unlock(&d->opd_pre_lock); /* * The locking above makes sure the opd_pre_reserved check below will * catch all osp_precreate_reserve() calls who find * "!opd_pre_recovering". */ l_wait_event(d->opd_pre_waitq, (!d->opd_pre_reserved && d->opd_recovery_completed) || !osp_precreate_running(d) || d->opd_got_disconnected, &lwi); if (!osp_precreate_running(d) || d->opd_got_disconnected) GOTO(out, rc = -EAGAIN); CDEBUG(D_HA, "%s: going to cleanup orphans since "DFID"\n", d->opd_obd->obd_name, PFID(&d->opd_last_used_fid)); *last_fid = d->opd_last_used_fid; /* The OSP should already get the valid seq now */ LASSERT(!fid_is_zero(last_fid)); if (fid_oid(&d->opd_last_used_fid) < 2) { /* lastfid looks strange... ask OST */ rc = osp_get_lastfid_from_ost(env, d); if (rc) GOTO(out, rc); } imp = d->opd_obd->u.cli.cl_import; LASSERT(imp); req = ptlrpc_request_alloc(imp, &RQF_OST_CREATE); if (req == NULL) GOTO(out, rc = -ENOMEM); rc = ptlrpc_request_pack(req, LUSTRE_OST_VERSION, OST_CREATE); if (rc) { ptlrpc_request_free(req); req = NULL; GOTO(out, rc); } body = req_capsule_client_get(&req->rq_pill, &RMF_OST_BODY); if (body == NULL) GOTO(out, rc = -EPROTO); body->oa.o_flags = OBD_FL_DELORPHAN; body->oa.o_valid = OBD_MD_FLFLAGS | OBD_MD_FLGROUP; fid_to_ostid(&d->opd_last_used_fid, &body->oa.o_oi); ptlrpc_request_set_replen(req); /* Don't resend the delorphan req */ req->rq_no_resend = req->rq_no_delay = 1; rc = ptlrpc_queue_wait(req); if (rc) { update_status = 1; GOTO(out, rc); } body = req_capsule_server_get(&req->rq_pill, &RMF_OST_BODY); if (body == NULL) GOTO(out, rc = -EPROTO); /* * OST provides us with id new pool starts from in body->oa.o_id */ ostid_to_fid(last_fid, &body->oa.o_oi, d->opd_index); spin_lock(&d->opd_pre_lock); diff = lu_fid_diff(&d->opd_last_used_fid, last_fid); if (diff > 0) { d->opd_pre_grow_count = OST_MIN_PRECREATE + diff; d->opd_pre_last_created_fid = d->opd_last_used_fid; } else { d->opd_pre_grow_count = OST_MIN_PRECREATE; d->opd_pre_last_created_fid = *last_fid; } /* * This empties the pre-creation pool and effectively blocks any new * reservations. */ LASSERT(fid_oid(&d->opd_pre_last_created_fid) <= LUSTRE_DATA_SEQ_MAX_WIDTH); d->opd_pre_used_fid = d->opd_pre_last_created_fid; d->opd_pre_grow_slow = 0; spin_unlock(&d->opd_pre_lock); CDEBUG(D_HA, "%s: Got last_id "DFID" from OST, last_created "DFID "last_used is "DFID"\n", d->opd_obd->obd_name, PFID(last_fid), PFID(&d->opd_pre_last_created_fid), PFID(&d->opd_last_used_fid)); out: if (req) ptlrpc_req_finished(req); spin_lock(&d->opd_pre_lock); d->opd_pre_recovering = 0; spin_unlock(&d->opd_pre_lock); /* * If rc is zero, the pre-creation window should have been emptied. * Since waking up the herd would be useless without pre-created * objects, we defer the signal to osp_precreate_send() in that case. */ if (rc != 0) { if (update_status) { CERROR("%s: cannot cleanup orphans: rc = %d\n", d->opd_obd->obd_name, rc); /* we can't proceed from here, OST seem to * be in a bad shape, better to wait for * a new instance of the server and repeat * from the beginning. notify possible waiters * this OSP isn't quite functional yet */ osp_pre_update_status(d, rc); } else { wake_up(&d->opd_pre_user_waitq); } } RETURN(rc); }