static int osp_wr_max_create_count(struct file *file, const char *buffer,
unsigned long count, void *data)
{
struct obd_device *obd = data;
struct osp_device *osp = lu2osp_dev(obd->obd_lu_dev);
int val, rc;
if (osp == NULL)
return 0;
rc = lprocfs_write_helper(buffer, count, &val);
if (rc)
return rc;
if (val < 0)
return -ERANGE;
if (val > OST_MAX_PRECREATE)
return -ERANGE;
if (osp->opd_pre_grow_count > val)
osp->opd_pre_grow_count = val;
osp->opd_pre_max_grow_count = val;
return count;
}
static int osp_wr_create_count(struct file *file, const char *buffer,
unsigned long count, void *data)
{
struct obd_device *obd = data;
struct osp_device *osp = lu2osp_dev(obd->obd_lu_dev);
int val, rc, i;
if (osp == NULL)
return 0;
rc = lprocfs_write_helper(buffer, count, &val);
if (rc)
return rc;
/* The MDT ALWAYS needs to limit the precreate count to
* OST_MAX_PRECREATE, and the constant cannot be changed
* because it is a value shared between the OSP and OST
* that is the maximum possible number of objects that will
* ever be handled by MDT->OST recovery processing.
*
* If the OST ever gets a request to delete more orphans,
* this implies that something has gone badly on the MDT
* and the OST will refuse to delete so much data from the
* filesystem as a safety measure. */
if (val < OST_MIN_PRECREATE || val > OST_MAX_PRECREATE)
return -ERANGE;
if (val > osp->opd_pre_max_grow_count)
return -ERANGE;
for (i = 1; (i << 1) <= val; i <<= 1)
;
osp->opd_pre_grow_count = i;
return count;
}
/**
* Implement OSP layer dt_object_operations::do_destroy() interface.
*
* Pack the destroy update into the RPC buffer, which will be sent
* to the remote MDT during transaction stop.
*
* It also marks the object as non-cached.
*
* \param[in] env pointer to the thread context
* \param[in] dt pointer to the OSP layer dt_object to be destroyed
* \param[in] th pointer to the transaction handler
*
* \retval 0 for success
* \retval negative error number on failure
*/
int osp_md_object_destroy(const struct lu_env *env, struct dt_object *dt,
struct thandle *th)
{
struct osp_object *o = dt2osp_obj(dt);
struct osp_device *osp = lu2osp_dev(dt->do_lu.lo_dev);
struct osp_update_request *update;
int rc = 0;
ENTRY;
o->opo_non_exist = 1;
LASSERT(osp->opd_connect_mdt);
update = thandle_to_osp_update_request(th);
LASSERT(update != NULL);
rc = osp_update_rpc_pack(env, object_destroy, update, OUT_DESTROY,
lu_object_fid(&dt->do_lu));
if (rc != 0)
RETURN(rc);
set_bit(LU_OBJECT_HEARD_BANSHEE, &dt->do_lu.lo_header->loh_flags);
rc = osp_insert_update_callback(env, update, dt2osp_obj(dt), NULL,
NULL);
RETURN(rc);
}
static int osp_declare_attr_set(const struct lu_env *env, struct dt_object *dt,
const struct lu_attr *attr, struct thandle *th)
{
struct osp_device *d = lu2osp_dev(dt->do_lu.lo_dev);
struct osp_object *o = dt2osp_obj(dt);
int rc = 0;
ENTRY;
/*
* Usually we don't allow server stack to manipulate size
* but there is a special case when striping is created
* late, after stripless file got truncated to non-zero.
*
* In this case we do the following:
*
* 1) grab id in declare - this can lead to leaked OST objects
* but we don't currently have proper mechanism and the only
* options we have are to do truncate RPC holding transaction
* open (very bad) or to grab id in declare at cost of leaked
* OST object in same very rare unfortunate case (just bad)
* notice 1.6-2.0 do assignment outside of running transaction
* all the time, meaning many more chances for leaked objects.
*
* 2) send synchronous truncate RPC with just assigned id
*/
/* there are few places in MDD code still passing NULL
* XXX: to be fixed soon */
if (attr == NULL)
RETURN(0);
if (attr->la_valid & LA_SIZE && attr->la_size > 0 &&
fid_is_zero(lu_object_fid(&o->opo_obj.do_lu))) {
LASSERT(!dt_object_exists(dt));
osp_object_assign_fid(env, d, o);
rc = osp_object_truncate(env, dt, attr->la_size);
if (rc)
RETURN(rc);
}
if (o->opo_new) {
/* no need in logging for new objects being created */
RETURN(0);
}
if (!(attr->la_valid & (LA_UID | LA_GID)))
RETURN(0);
/*
* track all UID/GID changes via llog
*/
rc = osp_sync_declare_add(env, o, MDS_SETATTR64_REC, th);
RETURN(rc);
}
static int osp_rd_max_create_count(char *page, char **start, off_t off,
int count, int *eof, void *data)
{
struct obd_device *obd = data;
struct osp_device *osp = lu2osp_dev(obd->obd_lu_dev);
if (osp == NULL)
return 0;
return snprintf(page, count, "%d\n", osp->opd_pre_max_grow_count);
}
static int osp_rd_prealloc_reserved(char *page, char **start, off_t off,
int count, int *eof, void *data)
{
struct obd_device *obd = data;
struct osp_device *osp = lu2osp_dev(obd->obd_lu_dev);
if (osp == NULL)
return 0;
return snprintf(page, count, LPU64"\n", osp->opd_pre_reserved);
}
static int osp_rd_prealloc_next_id(char *page, char **start, off_t off,
int count, int *eof, void *data)
{
struct obd_device *obd = data;
struct osp_device *osp = lu2osp_dev(obd->obd_lu_dev);
if (osp == NULL)
return 0;
return snprintf(page, count, "%u\n",
fid_oid(&osp->opd_pre_used_fid) + 1);
}
static int osp_rd_prealloc_last_seq(char *page, char **start, off_t off,
int count, int *eof, void *data)
{
struct obd_device *obd = data;
struct osp_device *osp = lu2osp_dev(obd->obd_lu_dev);
if (osp == NULL)
return 0;
return snprintf(page, count, LPX64"\n",
fid_seq(&osp->opd_pre_last_created_fid));
}
static int osp_rd_syn_in_prog(char *page, char **start, off_t off, int count,
int *eof, void *data)
{
struct obd_device *dev = data;
struct osp_device *osp = lu2osp_dev(dev->obd_lu_dev);
int rc;
if (osp == NULL)
return -EINVAL;
rc = snprintf(page, count, "%u\n", osp->opd_syn_rpc_in_progress);
return rc;
}
static int osp_rd_pre_status(char *page, char **start, off_t off,
int count, int *eof, void *data)
{
struct obd_device *dev = data;
struct osp_device *osp = lu2osp_dev(dev->obd_lu_dev);
int rc;
if (osp == NULL || osp->opd_pre == NULL)
return -EINVAL;
rc = snprintf(page, count, "%d\n", osp->opd_pre_status);
return rc;
}
static int osp_rd_old_sync_processed(char *page, char **start, off_t off,
int count, int *eof, void *data)
{
struct obd_device *dev = data;
struct osp_device *osp = lu2osp_dev(dev->obd_lu_dev);
int rc;
if (osp == NULL)
return -EINVAL;
rc = snprintf(page, count, "%d\n", osp->opd_syn_prev_done);
return rc;
}
static int osp_rd_destroys_in_flight(char *page, char **start, off_t off,
int count, int *eof, void *data)
{
struct obd_device *dev = data;
struct osp_device *osp = lu2osp_dev(dev->obd_lu_dev);
/*
* This counter used to determine if OST has space returned.
* Now we need to wait for the following:
* - sync changes are zero - no llog records
* - sync in progress are zero - no RPCs in flight
*/
return snprintf(page, count, "%lu\n",
osp->opd_syn_rpc_in_progress + osp->opd_syn_changes);
}
static int osp_wr_max_rpcs_in_flight(struct file *file, const char *buffer,
unsigned long count, void *data)
{
struct obd_device *dev = data;
struct osp_device *osp = lu2osp_dev(dev->obd_lu_dev);
int val, rc;
if (osp == NULL)
return -EINVAL;
rc = lprocfs_write_helper(buffer, count, &val);
if (rc)
return rc;
if (val < 1)
return -ERANGE;
osp->opd_syn_max_rpc_in_flight = val;
return count;
}
static void osp_object_release(const struct lu_env *env, struct lu_object *o)
{
struct osp_object *po = lu2osp_obj(o);
struct osp_device *d = lu2osp_dev(o->lo_dev);
ENTRY;
/*
* release reservation if object was declared but not created
* this may require lu_object_put() in LOD
*/
if (unlikely(po->opo_reserved)) {
LASSERT(d->opd_pre_reserved > 0);
spin_lock(&d->opd_pre_lock);
d->opd_pre_reserved--;
spin_unlock(&d->opd_pre_lock);
/* not needed in cache any more */
set_bit(LU_OBJECT_HEARD_BANSHEE, &o->lo_header->loh_flags);
}
EXIT;
}
static int is_ost_obj(struct lu_object *lo)
{
struct osp_device *osp = lu2osp_dev(lo->lo_dev);
return !osp->opd_connect_mdt;
}
static ssize_t osp_md_read(const struct lu_env *env, struct dt_object *dt,
struct lu_buf *rbuf, loff_t *pos)
{
struct osp_device *osp = lu2osp_dev(dt->do_lu.lo_dev);
struct dt_device *dt_dev = &osp->opd_dt_dev;
struct lu_buf *lbuf = &osp_env_info(env)->osi_lb2;
char *ptr = rbuf->lb_buf;
struct osp_update_request *update;
struct ptlrpc_request *req = NULL;
struct out_read_reply *orr;
struct ptlrpc_bulk_desc *desc;
struct object_update_reply *reply;
__u32 left_size;
int nbufs;
int i;
int rc;
ENTRY;
/* Because it needs send the update buffer right away,
* just create an update buffer, instead of attaching the
* update_remote list of the thandle. */
update = osp_update_request_create(dt_dev);
if (IS_ERR(update))
RETURN(PTR_ERR(update));
rc = osp_update_rpc_pack(env, read, update, OUT_READ,
lu_object_fid(&dt->do_lu),
rbuf->lb_len, *pos);
if (rc != 0) {
CERROR("%s: cannot insert update: rc = %d\n",
dt_dev->dd_lu_dev.ld_obd->obd_name, rc);
GOTO(out_update, rc);
}
CDEBUG(D_INFO, "%s "DFID" read offset %llu size %zu\n",
dt_dev->dd_lu_dev.ld_obd->obd_name,
PFID(lu_object_fid(&dt->do_lu)), *pos, rbuf->lb_len);
rc = osp_prep_update_req(env, osp->opd_obd->u.cli.cl_import, update,
&req);
if (rc != 0)
GOTO(out_update, rc);
nbufs = (rbuf->lb_len + OUT_BULK_BUFFER_SIZE - 1) /
OUT_BULK_BUFFER_SIZE;
/* allocate bulk descriptor */
desc = ptlrpc_prep_bulk_imp(req, nbufs, 1,
PTLRPC_BULK_PUT_SINK | PTLRPC_BULK_BUF_KVEC,
MDS_BULK_PORTAL, &ptlrpc_bulk_kvec_ops);
if (desc == NULL)
GOTO(out, rc = -ENOMEM);
/* split the buffer into small chunk size */
left_size = rbuf->lb_len;
for (i = 0; i < nbufs; i++) {
int read_size;
read_size = left_size > OUT_BULK_BUFFER_SIZE ?
OUT_BULK_BUFFER_SIZE : left_size;
desc->bd_frag_ops->add_iov_frag(desc, ptr, read_size);
ptr += read_size;
}
/* This will only be called with read-only update, and these updates
* might be used to retrieve update log during recovery process, so
* it will be allowed to send during recovery process */
req->rq_allow_replay = 1;
req->rq_bulk_read = 1;
/* send request to master and wait for RPC to complete */
rc = ptlrpc_queue_wait(req);
if (rc != 0)
GOTO(out, rc);
rc = sptlrpc_cli_unwrap_bulk_read(req, req->rq_bulk,
req->rq_bulk->bd_nob_transferred);
if (rc < 0)
GOTO(out, rc);
reply = req_capsule_server_sized_get(&req->rq_pill,
&RMF_OUT_UPDATE_REPLY,
OUT_UPDATE_REPLY_SIZE);
if (reply->ourp_magic != UPDATE_REPLY_MAGIC) {
CERROR("%s: invalid update reply magic %x expected %x:"
" rc = %d\n", dt_dev->dd_lu_dev.ld_obd->obd_name,
reply->ourp_magic, UPDATE_REPLY_MAGIC, -EPROTO);
GOTO(out, rc = -EPROTO);
}
rc = object_update_result_data_get(reply, lbuf, 0);
if (rc < 0)
GOTO(out, rc);
if (lbuf->lb_len < sizeof(*orr))
GOTO(out, rc = -EPROTO);
orr = lbuf->lb_buf;
orr_le_to_cpu(orr, orr);
rc = orr->orr_size;
*pos = orr->orr_offset;
out:
ptlrpc_req_finished(req);
out_update:
osp_update_request_destroy(update);
RETURN(rc);
}
/**
* Implementation of dt_index_operations::dio_lookup
*
* Look up record by key under a remote index object. It packs lookup update
* into RPC, sends to the remote OUT and waits for the lookup result.
*
* \param[in] env execution environment
* \param[in] dt index object to lookup
* \param[out] rec record in which to return lookup result
* \param[in] key key of index which will be looked up
*
* \retval 1 if the lookup succeeds.
* \retval negative errno if the lookup fails.
*/
static int osp_md_index_lookup(const struct lu_env *env, struct dt_object *dt,
struct dt_rec *rec, const struct dt_key *key)
{
struct lu_buf *lbuf = &osp_env_info(env)->osi_lb2;
struct osp_device *osp = lu2osp_dev(dt->do_lu.lo_dev);
struct dt_device *dt_dev = &osp->opd_dt_dev;
struct osp_update_request *update;
struct object_update_reply *reply;
struct ptlrpc_request *req = NULL;
struct lu_fid *fid;
int rc;
ENTRY;
/* Because it needs send the update buffer right away,
* just create an update buffer, instead of attaching the
* update_remote list of the thandle.
*/
update = osp_update_request_create(dt_dev);
if (IS_ERR(update))
RETURN(PTR_ERR(update));
rc = osp_update_rpc_pack(env, index_lookup, update, OUT_INDEX_LOOKUP,
lu_object_fid(&dt->do_lu), rec, key);
if (rc != 0) {
CERROR("%s: Insert update error: rc = %d\n",
dt_dev->dd_lu_dev.ld_obd->obd_name, rc);
GOTO(out, rc);
}
rc = osp_remote_sync(env, osp, update, &req);
if (rc < 0)
GOTO(out, rc);
reply = req_capsule_server_sized_get(&req->rq_pill,
&RMF_OUT_UPDATE_REPLY,
OUT_UPDATE_REPLY_SIZE);
if (reply->ourp_magic != UPDATE_REPLY_MAGIC) {
CERROR("%s: Wrong version %x expected %x: rc = %d\n",
dt_dev->dd_lu_dev.ld_obd->obd_name,
reply->ourp_magic, UPDATE_REPLY_MAGIC, -EPROTO);
GOTO(out, rc = -EPROTO);
}
rc = object_update_result_data_get(reply, lbuf, 0);
if (rc < 0)
GOTO(out, rc);
if (lbuf->lb_len != sizeof(*fid)) {
CERROR("%s: lookup "DFID" %s wrong size %d\n",
dt_dev->dd_lu_dev.ld_obd->obd_name,
PFID(lu_object_fid(&dt->do_lu)), (char *)key,
(int)lbuf->lb_len);
GOTO(out, rc = -EINVAL);
}
fid = lbuf->lb_buf;
if (ptlrpc_rep_need_swab(req))
lustre_swab_lu_fid(fid);
if (!fid_is_sane(fid)) {
CERROR("%s: lookup "DFID" %s invalid fid "DFID"\n",
dt_dev->dd_lu_dev.ld_obd->obd_name,
PFID(lu_object_fid(&dt->do_lu)), (char *)key, PFID(fid));
GOTO(out, rc = -EINVAL);
}
memcpy(rec, fid, sizeof(*fid));
GOTO(out, rc = 1);
out:
if (req != NULL)
ptlrpc_req_finished(req);
osp_update_request_destroy(update);
return rc;
}
static int osp_object_create(const struct lu_env *env, struct dt_object *dt,
struct lu_attr *attr,
struct dt_allocation_hint *hint,
struct dt_object_format *dof, struct thandle *th)
{
struct osp_thread_info *osi = osp_env_info(env);
struct osp_device *d = lu2osp_dev(dt->do_lu.lo_dev);
struct osp_object *o = dt2osp_obj(dt);
int rc = 0;
struct lu_fid *fid = &osi->osi_fid;
ENTRY;
if (o->opo_reserved) {
/* regular case, fid is assigned holding trunsaction open */
osp_object_assign_fid(env, d, o);
}
memcpy(fid, lu_object_fid(&dt->do_lu), sizeof(*fid));
LASSERTF(fid_is_sane(fid), "fid for osp_obj %p is insane"DFID"!\n",
osp_obj, PFID(fid));
if (!o->opo_reserved) {
/* special case, id was assigned outside of transaction
* see comments in osp_declare_attr_set */
spin_lock(&d->opd_pre_lock);
osp_update_last_fid(d, fid);
spin_unlock(&d->opd_pre_lock);
}
CDEBUG(D_INODE, "fid for osp_obj %p is "DFID"!\n", osp_obj, PFID(fid));
/* If the precreate ends, it means it will be ready to rollover to
* the new sequence soon, all the creation should be synchronized,
* otherwise during replay, the replay fid will be inconsistent with
* last_used/create fid */
if (osp_precreate_end_seq(env, d) && osp_is_fid_client(d))
th->th_sync = 1;
/*
* it's OK if the import is inactive by this moment - id was created
* by OST earlier, we just need to maintain it consistently on the disk
* once import is reconnected, OSP will claim this and other objects
* used and OST either keep them, if they exist or recreate
*/
/* we might have lost precreated objects */
if (unlikely(d->opd_gap_count) > 0) {
spin_lock(&d->opd_pre_lock);
if (d->opd_gap_count > 0) {
int count = d->opd_gap_count;
ostid_set_id(&osi->osi_oi,
fid_oid(&d->opd_gap_start_fid));
d->opd_gap_count = 0;
spin_unlock(&d->opd_pre_lock);
CDEBUG(D_HA, "Writting gap "DFID"+%d in llog\n",
PFID(&d->opd_gap_start_fid), count);
/* real gap handling is disabled intil ORI-692 will be
* fixed, now we only report gaps */
} else {
spin_unlock(&d->opd_pre_lock);
}
}
/* new object, the very first ->attr_set()
* initializing attributes needs no logging */
o->opo_new = 1;
/* Only need update last_used oid file, seq file will only be update
* during seq rollover */
osp_objid_buf_prep(&osi->osi_lb, &osi->osi_off,
&d->opd_last_used_fid.f_oid, d->opd_index);
rc = dt_record_write(env, d->opd_last_used_oid_file, &osi->osi_lb,
&osi->osi_off, th);
CDEBUG(D_HA, "%s: Wrote last used FID: "DFID", index %d: %d\n",
d->opd_obd->obd_name, PFID(fid), d->opd_index, rc);
RETURN(rc);
}
int osp_object_truncate(const struct lu_env *env, struct dt_object *dt,
__u64 size)
{
struct osp_device *d = lu2osp_dev(dt->do_lu.lo_dev);
struct ptlrpc_request *req = NULL;
struct obd_import *imp;
struct ost_body *body;
struct obdo *oa = NULL;
int rc;
ENTRY;
imp = d->opd_obd->u.cli.cl_import;
LASSERT(imp);
req = ptlrpc_request_alloc(imp, &RQF_OST_PUNCH);
if (req == NULL)
RETURN(-ENOMEM);
/* XXX: capa support? */
/* osc_set_capa_size(req, &RMF_CAPA1, capa); */
rc = ptlrpc_request_pack(req, LUSTRE_OST_VERSION, OST_PUNCH);
if (rc) {
ptlrpc_request_free(req);
RETURN(rc);
}
/*
* XXX: decide how do we do here with resend
* if we don't resend, then client may see wrong file size
* if we do resend, then MDS thread can get stuck for quite long
*/
req->rq_no_resend = req->rq_no_delay = 1;
req->rq_request_portal = OST_IO_PORTAL; /* bug 7198 */
ptlrpc_at_set_req_timeout(req);
OBD_ALLOC_PTR(oa);
if (oa == NULL)
GOTO(out, rc = -ENOMEM);
rc = fid_to_ostid(lu_object_fid(&dt->do_lu), &oa->o_oi);
LASSERT(rc == 0);
oa->o_size = size;
oa->o_blocks = OBD_OBJECT_EOF;
oa->o_valid = OBD_MD_FLSIZE | OBD_MD_FLBLOCKS |
OBD_MD_FLID | OBD_MD_FLGROUP;
body = req_capsule_client_get(&req->rq_pill, &RMF_OST_BODY);
LASSERT(body);
lustre_set_wire_obdo(&req->rq_import->imp_connect_data, &body->oa, oa);
/* XXX: capa support? */
/* osc_pack_capa(req, body, capa); */
ptlrpc_request_set_replen(req);
rc = ptlrpc_queue_wait(req);
if (rc)
CERROR("can't punch object: %d\n", rc);
out:
ptlrpc_req_finished(req);
if (oa)
OBD_FREE_PTR(oa);
RETURN(rc);
}
static int osp_declare_object_create(const struct lu_env *env,
struct dt_object *dt,
struct lu_attr *attr,
struct dt_allocation_hint *hint,
struct dt_object_format *dof,
struct thandle *th)
{
struct osp_thread_info *osi = osp_env_info(env);
struct osp_device *d = lu2osp_dev(dt->do_lu.lo_dev);
struct osp_object *o = dt2osp_obj(dt);
const struct lu_fid *fid;
int rc = 0;
ENTRY;
/* should happen to non-0 OSP only so that at least one object
* has been already declared in the scenario and LOD should
* cleanup that */
if (OBD_FAIL_CHECK(OBD_FAIL_MDS_OSC_CREATE_FAIL) && d->opd_index == 1)
RETURN(-ENOSPC);
LASSERT(d->opd_last_used_oid_file);
fid = lu_object_fid(&dt->do_lu);
/*
* There can be gaps in precreated ids and record to unlink llog
* XXX: we do not handle gaps yet, implemented before solution
* was found to be racy, so we disabled that. there is no
* point in making useless but expensive llog declaration.
*/
/* rc = osp_sync_declare_add(env, o, MDS_UNLINK64_REC, th); */
if (unlikely(!fid_is_zero(fid))) {
/* replay case: caller knows fid */
osi->osi_off = sizeof(osi->osi_id) * d->opd_index;
rc = dt_declare_record_write(env, d->opd_last_used_oid_file,
sizeof(osi->osi_id), osi->osi_off,
th);
RETURN(rc);
}
/*
* in declaration we need to reserve object so that we don't block
* awaiting precreation RPC to complete
*/
rc = osp_precreate_reserve(env, d);
/*
* we also need to declare update to local "last used id" file for
* recovery if object isn't used for a reason, we need to release
* reservation, this can be made in osd_object_release()
*/
if (rc == 0) {
/* mark id is reserved: in create we don't want to talk
* to OST */
LASSERT(o->opo_reserved == 0);
o->opo_reserved = 1;
/* common for all OSPs file hystorically */
osi->osi_off = sizeof(osi->osi_id) * d->opd_index;
rc = dt_declare_record_write(env, d->opd_last_used_oid_file,
sizeof(osi->osi_id), osi->osi_off,
th);
} else {
/* not needed in the cache anymore */
set_bit(LU_OBJECT_HEARD_BANSHEE,
&dt->do_lu.lo_header->loh_flags);
}
RETURN(rc);
}
