static int lustre_free_lsi(struct super_block *sb)
{
struct lustre_sb_info *lsi = s2lsi(sb);
LASSERT(lsi != NULL);
CDEBUG(D_MOUNT, "Freeing lsi %p\n", lsi);
/* someone didn't call server_put_mount. */
LASSERT(atomic_read(&lsi->lsi_mounts) == 0);
if (lsi->lsi_lmd != NULL) {
kfree(lsi->lsi_lmd->lmd_dev);
kfree(lsi->lsi_lmd->lmd_profile);
kfree(lsi->lsi_lmd->lmd_mgssec);
kfree(lsi->lsi_lmd->lmd_opts);
if (lsi->lsi_lmd->lmd_exclude_count)
kfree(lsi->lsi_lmd->lmd_exclude);
kfree(lsi->lsi_lmd->lmd_mgs);
kfree(lsi->lsi_lmd->lmd_osd_type);
kfree(lsi->lsi_lmd->lmd_params);
kfree(lsi->lsi_lmd);
}
LASSERT(lsi->lsi_llsbi == NULL);
kfree(lsi);
s2lsi_nocast(sb) = NULL;
return 0;
}
/* The lsi has one reference for every server that is using the disk -
* e.g. MDT, MGS, and potentially MGC
*/
static int lustre_put_lsi(struct super_block *sb)
{
struct lustre_sb_info *lsi = s2lsi(sb);
CDEBUG(D_MOUNT, "put %p %d\n", sb, atomic_read(&lsi->lsi_mounts));
if (atomic_dec_and_test(&lsi->lsi_mounts)) {
lustre_free_lsi(sb);
return 1;
}
return 0;
}
/** Get a config log from the MGS and process it.
* This func is called for both clients and servers.
* Continue to process new statements appended to the logs
* (whenever the config lock is revoked) until lustre_end_log
* is called.
* @param sb The superblock is used by the MGC to write to the local copy of
* the config log
* @param logname The name of the llog to replicate from the MGS
* @param cfg Since the same mgc may be used to follow multiple config logs
* (e.g. ost1, ost2, client), the config_llog_instance keeps the state for
* this log, and is added to the mgc's list of logs to follow.
*/
int lustre_process_log(struct super_block *sb, char *logname,
struct config_llog_instance *cfg)
{
struct lustre_cfg *lcfg;
struct lustre_cfg_bufs *bufs;
struct lustre_sb_info *lsi = s2lsi(sb);
struct obd_device *mgc = lsi->lsi_mgc;
int rc;
LASSERT(mgc);
LASSERT(cfg);
OBD_ALLOC_PTR(bufs);
if (bufs == NULL)
return -ENOMEM;
/* mgc_process_config */
lustre_cfg_bufs_reset(bufs, mgc->obd_name);
lustre_cfg_bufs_set_string(bufs, 1, logname);
lustre_cfg_bufs_set(bufs, 2, cfg, sizeof(*cfg));
lustre_cfg_bufs_set(bufs, 3, &sb, sizeof(sb));
lcfg = lustre_cfg_new(LCFG_LOG_START, bufs);
rc = obd_process_config(mgc, sizeof(*lcfg), lcfg);
lustre_cfg_free(lcfg);
OBD_FREE_PTR(bufs);
if (rc == -EINVAL)
LCONSOLE_ERROR_MSG(0x15b, "%s: The configuration from log '%s'"
"failed from the MGS (%d). Make sure this "
"client and the MGS are running compatible "
"versions of Lustre.\n",
mgc->obd_name, logname, rc);
if (rc)
LCONSOLE_ERROR_MSG(0x15c, "%s: The configuration from log '%s' "
"failed (%d). This may be the result of "
"communication errors between this node and "
"the MGS, a bad configuration, or other "
"errors. See the syslog for more "
"information.\n", mgc->obd_name, logname,
rc);
/* class_obd_list(); */
return rc;
}
static int lustre_free_lsi(struct super_block *sb)
{
struct lustre_sb_info *lsi = s2lsi(sb);
LASSERT(lsi != NULL);
CDEBUG(D_MOUNT, "Freeing lsi %p\n", lsi);
/* someone didn't call server_put_mount. */
LASSERT(atomic_read(&lsi->lsi_mounts) == 0);
if (lsi->lsi_lmd != NULL) {
if (lsi->lsi_lmd->lmd_dev != NULL)
OBD_FREE(lsi->lsi_lmd->lmd_dev,
strlen(lsi->lsi_lmd->lmd_dev) + 1);
if (lsi->lsi_lmd->lmd_profile != NULL)
OBD_FREE(lsi->lsi_lmd->lmd_profile,
strlen(lsi->lsi_lmd->lmd_profile) + 1);
if (lsi->lsi_lmd->lmd_mgssec != NULL)
OBD_FREE(lsi->lsi_lmd->lmd_mgssec,
strlen(lsi->lsi_lmd->lmd_mgssec) + 1);
if (lsi->lsi_lmd->lmd_opts != NULL)
OBD_FREE(lsi->lsi_lmd->lmd_opts,
strlen(lsi->lsi_lmd->lmd_opts) + 1);
if (lsi->lsi_lmd->lmd_exclude_count)
OBD_FREE(lsi->lsi_lmd->lmd_exclude,
sizeof(lsi->lsi_lmd->lmd_exclude[0]) *
lsi->lsi_lmd->lmd_exclude_count);
if (lsi->lsi_lmd->lmd_mgs != NULL)
OBD_FREE(lsi->lsi_lmd->lmd_mgs,
strlen(lsi->lsi_lmd->lmd_mgs) + 1);
if (lsi->lsi_lmd->lmd_osd_type != NULL)
OBD_FREE(lsi->lsi_lmd->lmd_osd_type,
strlen(lsi->lsi_lmd->lmd_osd_type) + 1);
if (lsi->lsi_lmd->lmd_params != NULL)
OBD_FREE(lsi->lsi_lmd->lmd_params, 4096);
OBD_FREE(lsi->lsi_lmd, sizeof(*lsi->lsi_lmd));
}
LASSERT(lsi->lsi_llsbi == NULL);
OBD_FREE(lsi, sizeof(*lsi));
s2lsi_nocast(sb) = NULL;
return 0;
}
/* The lsi has one reference for every server that is using the disk -
e.g. MDT, MGS, and potentially MGC */
int lustre_put_lsi(struct super_block *sb)
{
struct lustre_sb_info *lsi = s2lsi(sb);
LASSERT(lsi != NULL);
CDEBUG(D_MOUNT, "put %p %d\n", sb, atomic_read(&lsi->lsi_mounts));
if (atomic_dec_and_test(&lsi->lsi_mounts)) {
if (IS_SERVER(lsi) && lsi->lsi_osd_exp) {
lu_device_put(&lsi->lsi_dt_dev->dd_lu_dev);
lsi->lsi_osd_exp->exp_obd->obd_lvfs_ctxt.dt = NULL;
lsi->lsi_dt_dev = NULL;
obd_disconnect(lsi->lsi_osd_exp);
/* wait till OSD is gone */
obd_zombie_barrier();
}
lustre_free_lsi(sb);
return 1;
}
return 0;
}
/* Stop watching this config log for updates */
int lustre_end_log(struct super_block *sb, char *logname,
struct config_llog_instance *cfg)
{
struct lustre_cfg *lcfg;
struct lustre_cfg_bufs bufs;
struct lustre_sb_info *lsi = s2lsi(sb);
struct obd_device *mgc = lsi->lsi_mgc;
int rc;
if (!mgc)
return -ENOENT;
/* mgc_process_config */
lustre_cfg_bufs_reset(&bufs, mgc->obd_name);
lustre_cfg_bufs_set_string(&bufs, 1, logname);
if (cfg)
lustre_cfg_bufs_set(&bufs, 2, cfg, sizeof(*cfg));
lcfg = lustre_cfg_new(LCFG_LOG_END, &bufs);
rc = obd_process_config(mgc, sizeof(*lcfg), lcfg);
lustre_cfg_free(lcfg);
return rc;
}
/* Is this server on the exclusion list */
int lustre_check_exclusion(struct super_block *sb, char *svname)
{
struct lustre_sb_info *lsi = s2lsi(sb);
struct lustre_mount_data *lmd = lsi->lsi_lmd;
__u32 index;
int i, rc;
rc = server_name2index(svname, &index, NULL);
if (rc != LDD_F_SV_TYPE_OST)
/* Only exclude OSTs */
return 0;
CDEBUG(D_MOUNT, "Check exclusion %s (%d) in %d of %s\n", svname,
index, lmd->lmd_exclude_count, lmd->lmd_dev);
for(i = 0; i < lmd->lmd_exclude_count; i++) {
if (index == lmd->lmd_exclude[i]) {
CWARN("Excluding %s (on exclusion list)\n", svname);
return 1;
}
}
return 0;
}
static int lustre_stop_mgc(struct super_block *sb)
{
struct lustre_sb_info *lsi = s2lsi(sb);
struct obd_device *obd;
char *niduuid = 0, *ptr = 0;
int i, rc = 0, len = 0;
if (!lsi)
return -ENOENT;
obd = lsi->lsi_mgc;
if (!obd)
return -ENOENT;
lsi->lsi_mgc = NULL;
mutex_lock(&mgc_start_lock);
LASSERT(atomic_read(&obd->u.cli.cl_mgc_refcount) > 0);
if (!atomic_dec_and_test(&obd->u.cli.cl_mgc_refcount)) {
/* This is not fatal, every client that stops
will call in here. */
CDEBUG(D_MOUNT, "mgc still has %d references.\n",
atomic_read(&obd->u.cli.cl_mgc_refcount));
GOTO(out, rc = -EBUSY);
}
/* The MGC has no recoverable data in any case.
* force shotdown set in umount_begin */
obd->obd_no_recov = 1;
if (obd->u.cli.cl_mgc_mgsexp) {
/* An error is not fatal, if we are unable to send the
disconnect mgs ping evictor cleans up the export */
rc = obd_disconnect(obd->u.cli.cl_mgc_mgsexp);
if (rc)
CDEBUG(D_MOUNT, "disconnect failed %d\n", rc);
}
/* Save the obdname for cleaning the nid uuids, which are
obdname_XX */
len = strlen(obd->obd_name) + 6;
OBD_ALLOC(niduuid, len);
if (niduuid) {
strcpy(niduuid, obd->obd_name);
ptr = niduuid + strlen(niduuid);
}
rc = class_manual_cleanup(obd);
if (rc)
GOTO(out, rc);
/* Clean the nid uuids */
if (!niduuid)
GOTO(out, rc = -ENOMEM);
for (i = 0; i < lsi->lsi_lmd->lmd_mgs_failnodes; i++) {
sprintf(ptr, "_%x", i);
rc = do_lcfg(LUSTRE_MGC_OBDNAME, 0, LCFG_DEL_UUID,
niduuid, 0, 0, 0);
if (rc)
CERROR("del MDC UUID %s failed: rc = %d\n",
niduuid, rc);
}
out:
if (niduuid)
OBD_FREE(niduuid, len);
/* class_import_put will get rid of the additional connections */
mutex_unlock(&mgc_start_lock);
return rc;
}
/** Set up a mgc obd to process startup logs
*
* \param sb [in] super block of the mgc obd
*
* \retval 0 success, otherwise error code
*/
int lustre_start_mgc(struct super_block *sb)
{
struct obd_connect_data *data = NULL;
struct lustre_sb_info *lsi = s2lsi(sb);
struct obd_device *obd;
struct obd_export *exp;
struct obd_uuid *uuid;
class_uuid_t uuidc;
lnet_nid_t nid;
char *mgcname = NULL, *niduuid = NULL, *mgssec = NULL;
char *ptr;
int recov_bk;
int rc = 0, i = 0, j, len;
LASSERT(lsi->lsi_lmd);
/* Find the first non-lo MGS nid for our MGC name */
if (IS_SERVER(lsi)) {
/* mount -o mgsnode=nid */
ptr = lsi->lsi_lmd->lmd_mgs;
if (lsi->lsi_lmd->lmd_mgs &&
(class_parse_nid(lsi->lsi_lmd->lmd_mgs, &nid, &ptr) == 0)) {
i++;
} else if (IS_MGS(lsi)) {
lnet_process_id_t id;
while ((rc = LNetGetId(i++, &id)) != -ENOENT) {
if (LNET_NETTYP(LNET_NIDNET(id.nid)) == LOLND)
continue;
nid = id.nid;
i++;
break;
}
}
} else { /* client */
/* Use nids from mount line: uml1,1@elan:uml2,2@elan:/lustre */
ptr = lsi->lsi_lmd->lmd_dev;
if (class_parse_nid(ptr, &nid, &ptr) == 0)
i++;
}
if (i == 0) {
CERROR("No valid MGS nids found.\n");
return -EINVAL;
}
mutex_lock(&mgc_start_lock);
len = strlen(LUSTRE_MGC_OBDNAME) + strlen(libcfs_nid2str(nid)) + 1;
OBD_ALLOC(mgcname, len);
OBD_ALLOC(niduuid, len + 2);
if (!mgcname || !niduuid)
GOTO(out_free, rc = -ENOMEM);
sprintf(mgcname, "%s%s", LUSTRE_MGC_OBDNAME, libcfs_nid2str(nid));
mgssec = lsi->lsi_lmd->lmd_mgssec ? lsi->lsi_lmd->lmd_mgssec : "";
OBD_ALLOC_PTR(data);
if (data == NULL)
GOTO(out_free, rc = -ENOMEM);
obd = class_name2obd(mgcname);
if (obd && !obd->obd_stopping) {
rc = obd_set_info_async(NULL, obd->obd_self_export,
strlen(KEY_MGSSEC), KEY_MGSSEC,
strlen(mgssec), mgssec, NULL);
if (rc)
GOTO(out_free, rc);
/* Re-using an existing MGC */
atomic_inc(&obd->u.cli.cl_mgc_refcount);
/* IR compatibility check, only for clients */
if (lmd_is_client(lsi->lsi_lmd)) {
int has_ir;
int vallen = sizeof(*data);
__u32 *flags = &lsi->lsi_lmd->lmd_flags;
rc = obd_get_info(NULL, obd->obd_self_export,
strlen(KEY_CONN_DATA), KEY_CONN_DATA,
&vallen, data, NULL);
LASSERT(rc == 0);
has_ir = OCD_HAS_FLAG(data, IMP_RECOV);
if (has_ir ^ !(*flags & LMD_FLG_NOIR)) {
/* LMD_FLG_NOIR is for test purpose only */
LCONSOLE_WARN(
"Trying to mount a client with IR setting "
"not compatible with current mgc. "
"Force to use current mgc setting that is "
"IR %s.\n",
has_ir ? "enabled" : "disabled");
if (has_ir)
*flags &= ~LMD_FLG_NOIR;
else
*flags |= LMD_FLG_NOIR;
}
}
recov_bk = 0;
/* If we are restarting the MGS, don't try to keep the MGC's
old connection, or registration will fail. */
if (IS_MGS(lsi)) {
CDEBUG(D_MOUNT, "New MGS with live MGC\n");
recov_bk = 1;
}
/* Try all connections, but only once (again).
We don't want to block another target from starting
(using its local copy of the log), but we do want to connect
if at all possible. */
recov_bk++;
CDEBUG(D_MOUNT, "%s: Set MGC reconnect %d\n", mgcname,recov_bk);
rc = obd_set_info_async(NULL, obd->obd_self_export,
sizeof(KEY_INIT_RECOV_BACKUP),
KEY_INIT_RECOV_BACKUP,
sizeof(recov_bk), &recov_bk, NULL);
GOTO(out, rc = 0);
}
CDEBUG(D_MOUNT, "Start MGC '%s'\n", mgcname);
/* Add the primary nids for the MGS */
i = 0;
sprintf(niduuid, "%s_%x", mgcname, i);
if (IS_SERVER(lsi)) {
ptr = lsi->lsi_lmd->lmd_mgs;
if (IS_MGS(lsi)) {
/* Use local nids (including LO) */
lnet_process_id_t id;
while ((rc = LNetGetId(i++, &id)) != -ENOENT) {
rc = do_lcfg(mgcname, id.nid,
LCFG_ADD_UUID, niduuid, 0,0,0);
}
} else {
/* Use mgsnode= nids */
/* mount -o mgsnode=nid */
if (lsi->lsi_lmd->lmd_mgs) {
ptr = lsi->lsi_lmd->lmd_mgs;
} else if (class_find_param(ptr, PARAM_MGSNODE,
&ptr) != 0) {
CERROR("No MGS nids given.\n");
GOTO(out_free, rc = -EINVAL);
}
while (class_parse_nid(ptr, &nid, &ptr) == 0) {
rc = do_lcfg(mgcname, nid,
LCFG_ADD_UUID, niduuid, 0,0,0);
i++;
}
}
} else { /* client */
/* Use nids from mount line: uml1,1@elan:uml2,2@elan:/lustre */
ptr = lsi->lsi_lmd->lmd_dev;
while (class_parse_nid(ptr, &nid, &ptr) == 0) {
rc = do_lcfg(mgcname, nid,
LCFG_ADD_UUID, niduuid, 0,0,0);
i++;
/* Stop at the first failover nid */
if (*ptr == ':')
break;
}
}
if (i == 0) {
CERROR("No valid MGS nids found.\n");
GOTO(out_free, rc = -EINVAL);
}
lsi->lsi_lmd->lmd_mgs_failnodes = 1;
/* Random uuid for MGC allows easier reconnects */
OBD_ALLOC_PTR(uuid);
ll_generate_random_uuid(uuidc);
class_uuid_unparse(uuidc, uuid);
/* Start the MGC */
rc = lustre_start_simple(mgcname, LUSTRE_MGC_NAME,
(char *)uuid->uuid, LUSTRE_MGS_OBDNAME,
niduuid, 0, 0);
OBD_FREE_PTR(uuid);
if (rc)
GOTO(out_free, rc);
/* Add any failover MGS nids */
i = 1;
while (ptr && ((*ptr == ':' ||
class_find_param(ptr, PARAM_MGSNODE, &ptr) == 0))) {
/* New failover node */
sprintf(niduuid, "%s_%x", mgcname, i);
j = 0;
while (class_parse_nid_quiet(ptr, &nid, &ptr) == 0) {
j++;
rc = do_lcfg(mgcname, nid,
LCFG_ADD_UUID, niduuid, 0,0,0);
if (*ptr == ':')
break;
}
if (j > 0) {
rc = do_lcfg(mgcname, 0, LCFG_ADD_CONN,
niduuid, 0, 0, 0);
i++;
} else {
/* at ":/fsname" */
break;
}
}
lsi->lsi_lmd->lmd_mgs_failnodes = i;
obd = class_name2obd(mgcname);
if (!obd) {
CERROR("Can't find mgcobd %s\n", mgcname);
GOTO(out_free, rc = -ENOTCONN);
}
rc = obd_set_info_async(NULL, obd->obd_self_export,
strlen(KEY_MGSSEC), KEY_MGSSEC,
strlen(mgssec), mgssec, NULL);
if (rc)
GOTO(out_free, rc);
/* Keep a refcount of servers/clients who started with "mount",
so we know when we can get rid of the mgc. */
atomic_set(&obd->u.cli.cl_mgc_refcount, 1);
/* Try all connections, but only once. */
recov_bk = 1;
rc = obd_set_info_async(NULL, obd->obd_self_export,
sizeof(KEY_INIT_RECOV_BACKUP),
KEY_INIT_RECOV_BACKUP,
sizeof(recov_bk), &recov_bk, NULL);
if (rc)
/* nonfatal */
CWARN("can't set %s %d\n", KEY_INIT_RECOV_BACKUP, rc);
/* We connect to the MGS at setup, and don't disconnect until cleanup */
data->ocd_connect_flags = OBD_CONNECT_VERSION | OBD_CONNECT_AT |
OBD_CONNECT_FULL20 | OBD_CONNECT_IMP_RECOV |
OBD_CONNECT_LVB_TYPE;
#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(3, 2, 50, 0)
data->ocd_connect_flags |= OBD_CONNECT_MNE_SWAB;
#else
#warning "LU-1644: Remove old OBD_CONNECT_MNE_SWAB fixup and imp_need_mne_swab"
#endif
if (lmd_is_client(lsi->lsi_lmd) &&
lsi->lsi_lmd->lmd_flags & LMD_FLG_NOIR)
data->ocd_connect_flags &= ~OBD_CONNECT_IMP_RECOV;
data->ocd_version = LUSTRE_VERSION_CODE;
rc = obd_connect(NULL, &exp, obd, &(obd->obd_uuid), data, NULL);
if (rc) {
CERROR("connect failed %d\n", rc);
GOTO(out, rc);
}
obd->u.cli.cl_mgc_mgsexp = exp;
out:
/* Keep the mgc info in the sb. Note that many lsi's can point
to the same mgc.*/
lsi->lsi_mgc = obd;
out_free:
mutex_unlock(&mgc_start_lock);
if (data)
OBD_FREE_PTR(data);
if (mgcname)
OBD_FREE(mgcname, len);
if (niduuid)
OBD_FREE(niduuid, len + 2);
return rc;
}
/** Set up a mgc obd to process startup logs
*
* \param sb [in] super block of the mgc obd
*
* \retval 0 success, otherwise error code
*/
int lustre_start_mgc(struct super_block *sb)
{
struct obd_connect_data *data = NULL;
struct lustre_sb_info *lsi = s2lsi(sb);
struct obd_device *obd;
struct obd_export *exp;
struct obd_uuid *uuid;
class_uuid_t uuidc;
lnet_nid_t nid;
char nidstr[LNET_NIDSTR_SIZE];
char *mgcname = NULL, *niduuid = NULL, *mgssec = NULL;
char *ptr;
int rc = 0, i = 0, j;
LASSERT(lsi->lsi_lmd);
/* Use nids from mount line: uml1,1@elan:uml2,2@elan:/lustre */
ptr = lsi->lsi_lmd->lmd_dev;
if (class_parse_nid(ptr, &nid, &ptr) == 0)
i++;
if (i == 0) {
CERROR("No valid MGS nids found.\n");
return -EINVAL;
}
mutex_lock(&mgc_start_lock);
libcfs_nid2str_r(nid, nidstr, sizeof(nidstr));
mgcname = kasprintf(GFP_NOFS,
"%s%s", LUSTRE_MGC_OBDNAME, nidstr);
niduuid = kasprintf(GFP_NOFS, "%s_%x", mgcname, i);
if (!mgcname || !niduuid) {
rc = -ENOMEM;
goto out_free;
}
mgssec = lsi->lsi_lmd->lmd_mgssec ? lsi->lsi_lmd->lmd_mgssec : "";
data = kzalloc(sizeof(*data), GFP_NOFS);
if (!data) {
rc = -ENOMEM;
goto out_free;
}
obd = class_name2obd(mgcname);
if (obd && !obd->obd_stopping) {
int recov_bk;
rc = obd_set_info_async(NULL, obd->obd_self_export,
strlen(KEY_MGSSEC), KEY_MGSSEC,
strlen(mgssec), mgssec, NULL);
if (rc)
goto out_free;
/* Re-using an existing MGC */
atomic_inc(&obd->u.cli.cl_mgc_refcount);
/* IR compatibility check, only for clients */
if (lmd_is_client(lsi->lsi_lmd)) {
int has_ir;
int vallen = sizeof(*data);
__u32 *flags = &lsi->lsi_lmd->lmd_flags;
rc = obd_get_info(NULL, obd->obd_self_export,
strlen(KEY_CONN_DATA), KEY_CONN_DATA,
&vallen, data, NULL);
LASSERT(rc == 0);
has_ir = OCD_HAS_FLAG(data, IMP_RECOV);
if (has_ir ^ !(*flags & LMD_FLG_NOIR)) {
/* LMD_FLG_NOIR is for test purpose only */
LCONSOLE_WARN(
"Trying to mount a client with IR setting not compatible with current mgc. Force to use current mgc setting that is IR %s.\n",
has_ir ? "enabled" : "disabled");
if (has_ir)
*flags &= ~LMD_FLG_NOIR;
else
*flags |= LMD_FLG_NOIR;
}
}
recov_bk = 0;
/* Try all connections, but only once (again).
We don't want to block another target from starting
(using its local copy of the log), but we do want to connect
if at all possible. */
recov_bk++;
CDEBUG(D_MOUNT, "%s: Set MGC reconnect %d\n", mgcname,
recov_bk);
rc = obd_set_info_async(NULL, obd->obd_self_export,
sizeof(KEY_INIT_RECOV_BACKUP),
KEY_INIT_RECOV_BACKUP,
sizeof(recov_bk), &recov_bk, NULL);
rc = 0;
goto out;
}
CDEBUG(D_MOUNT, "Start MGC '%s'\n", mgcname);
/* Add the primary nids for the MGS */
i = 0;
/* Use nids from mount line: uml1,1@elan:uml2,2@elan:/lustre */
ptr = lsi->lsi_lmd->lmd_dev;
while (class_parse_nid(ptr, &nid, &ptr) == 0) {
rc = do_lcfg(mgcname, nid,
LCFG_ADD_UUID, niduuid, NULL, NULL, NULL);
i++;
/* Stop at the first failover nid */
if (*ptr == ':')
break;
}
if (i == 0) {
CERROR("No valid MGS nids found.\n");
rc = -EINVAL;
goto out_free;
}
lsi->lsi_lmd->lmd_mgs_failnodes = 1;
/* Random uuid for MGC allows easier reconnects */
uuid = kzalloc(sizeof(*uuid), GFP_NOFS);
if (!uuid) {
rc = -ENOMEM;
goto out_free;
}
ll_generate_random_uuid(uuidc);
class_uuid_unparse(uuidc, uuid);
/* Start the MGC */
rc = lustre_start_simple(mgcname, LUSTRE_MGC_NAME,
(char *)uuid->uuid, LUSTRE_MGS_OBDNAME,
niduuid, NULL, NULL);
kfree(uuid);
if (rc)
goto out_free;
/* Add any failover MGS nids */
i = 1;
while (ptr && ((*ptr == ':' ||
class_find_param(ptr, PARAM_MGSNODE, &ptr) == 0))) {
/* New failover node */
sprintf(niduuid, "%s_%x", mgcname, i);
j = 0;
while (class_parse_nid_quiet(ptr, &nid, &ptr) == 0) {
j++;
rc = do_lcfg(mgcname, nid,
LCFG_ADD_UUID, niduuid, NULL, NULL, NULL);
if (*ptr == ':')
break;
}
if (j > 0) {
rc = do_lcfg(mgcname, 0, LCFG_ADD_CONN,
niduuid, NULL, NULL, NULL);
i++;
} else {
/* at ":/fsname" */
break;
}
}
lsi->lsi_lmd->lmd_mgs_failnodes = i;
obd = class_name2obd(mgcname);
if (!obd) {
CERROR("Can't find mgcobd %s\n", mgcname);
rc = -ENOTCONN;
goto out_free;
}
rc = obd_set_info_async(NULL, obd->obd_self_export,
strlen(KEY_MGSSEC), KEY_MGSSEC,
strlen(mgssec), mgssec, NULL);
if (rc)
goto out_free;
/* Keep a refcount of servers/clients who started with "mount",
so we know when we can get rid of the mgc. */
atomic_set(&obd->u.cli.cl_mgc_refcount, 1);
/* We connect to the MGS at setup, and don't disconnect until cleanup */
data->ocd_connect_flags = OBD_CONNECT_VERSION | OBD_CONNECT_AT |
OBD_CONNECT_FULL20 | OBD_CONNECT_IMP_RECOV |
OBD_CONNECT_LVB_TYPE;
#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(3, 2, 50, 0)
data->ocd_connect_flags |= OBD_CONNECT_MNE_SWAB;
#else
#warning "LU-1644: Remove old OBD_CONNECT_MNE_SWAB fixup and imp_need_mne_swab"
#endif
if (lmd_is_client(lsi->lsi_lmd) &&
lsi->lsi_lmd->lmd_flags & LMD_FLG_NOIR)
data->ocd_connect_flags &= ~OBD_CONNECT_IMP_RECOV;
data->ocd_version = LUSTRE_VERSION_CODE;
rc = obd_connect(NULL, &exp, obd, &(obd->obd_uuid), data, NULL);
if (rc) {
CERROR("connect failed %d\n", rc);
goto out;
}
obd->u.cli.cl_mgc_mgsexp = exp;
out:
/* Keep the mgc info in the sb. Note that many lsi's can point
to the same mgc.*/
lsi->lsi_mgc = obd;
out_free:
mutex_unlock(&mgc_start_lock);
kfree(data);
kfree(mgcname);
kfree(niduuid);
return rc;
}
/*mds still need lov setup here*/
static int mds_cmd_setup(struct obd_device *obd, struct lustre_cfg *lcfg)
{
struct mds_obd *mds = &obd->u.mds;
struct lvfs_run_ctxt saved;
const char *dev;
struct vfsmount *mnt;
struct lustre_sb_info *lsi;
struct lustre_mount_info *lmi;
struct dentry *dentry;
int rc = 0;
ENTRY;
CDEBUG(D_INFO, "obd %s setup \n", obd->obd_name);
if (strncmp(obd->obd_name, MDD_OBD_NAME, strlen(MDD_OBD_NAME)))
RETURN(0);
if (lcfg->lcfg_bufcount < 5) {
CERROR("invalid arg for setup %s\n", MDD_OBD_NAME);
RETURN(-EINVAL);
}
dev = lustre_cfg_string(lcfg, 4);
lmi = server_get_mount(dev);
LASSERT(lmi != NULL);
lsi = s2lsi(lmi->lmi_sb);
mnt = lmi->lmi_mnt;
/* FIXME: MDD LOV initialize objects.
* we need only lmi here but not get mount
* OSD did mount already, so put mount back
*/
cfs_atomic_dec(&lsi->lsi_mounts);
mntput(mnt);
cfs_init_rwsem(&mds->mds_notify_lock);
obd->obd_fsops = fsfilt_get_ops(MT_STR(lsi->lsi_ldd));
mds_init_ctxt(obd, mnt);
push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
dentry = simple_mkdir(current->fs->pwd, mnt, "OBJECTS", 0777, 1);
if (IS_ERR(dentry)) {
rc = PTR_ERR(dentry);
CERROR("cannot create OBJECTS directory: rc = %d\n", rc);
GOTO(err_putfs, rc);
}
mds->mds_objects_dir = dentry;
dentry = ll_lookup_one_len("__iopen__", current->fs->pwd,
strlen("__iopen__"));
if (IS_ERR(dentry)) {
rc = PTR_ERR(dentry);
CERROR("cannot lookup __iopen__ directory: rc = %d\n", rc);
GOTO(err_objects, rc);
}
mds->mds_fid_de = dentry;
if (!dentry->d_inode || is_bad_inode(dentry->d_inode)) {
rc = -ENOENT;
CERROR("__iopen__ directory has no inode? rc = %d\n", rc);
GOTO(err_fid, rc);
}
rc = mds_lov_init_objids(obd);
if (rc != 0) {
CERROR("cannot init lov objid rc = %d\n", rc);
GOTO(err_fid, rc );
}
rc = mds_lov_presetup(mds, lcfg);
if (rc < 0)
GOTO(err_objects, rc);
/* Don't wait for mds_postrecov trying to clear orphans */
obd->obd_async_recov = 1;
rc = mds_postsetup(obd);
/* Bug 11557 - allow async abort_recov start
FIXME can remove most of this obd_async_recov plumbing
obd->obd_async_recov = 0;
*/
if (rc)
GOTO(err_objects, rc);
err_pop:
pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
RETURN(rc);
err_fid:
dput(mds->mds_fid_de);
err_objects:
dput(mds->mds_objects_dir);
err_putfs:
fsfilt_put_ops(obd->obd_fsops);
goto err_pop;
}
/** Set up a mgc obd to process startup logs
*
* \param sb [in] super block of the mgc obd
*
* \retval 0 success, otherwise error code
*/
int lustre_start_mgc(struct super_block *sb)
{
struct obd_connect_data *data = NULL;
struct lustre_sb_info *lsi = s2lsi(sb);
struct obd_device *obd;
struct obd_export *exp;
struct obd_uuid *uuid;
class_uuid_t uuidc;
lnet_nid_t nid;
char *mgcname = NULL, *niduuid = NULL, *mgssec = NULL;
char *ptr;
int rc = 0, i = 0, j, len;
LASSERT(lsi->lsi_lmd);
/* Find the first non-lo MGS nid for our MGC name */
if (IS_SERVER(lsi)) {
/* mount -o mgsnode=nid */
ptr = lsi->lsi_lmd->lmd_mgs;
if (lsi->lsi_lmd->lmd_mgs &&
(class_parse_nid(lsi->lsi_lmd->lmd_mgs, &nid, &ptr) == 0)) {
i++;
} else if (IS_MGS(lsi)) {
lnet_process_id_t id;
while ((rc = LNetGetId(i++, &id)) != -ENOENT) {
if (LNET_NETTYP(LNET_NIDNET(id.nid)) == LOLND)
continue;
nid = id.nid;
i++;
break;
}
}
} else { /* client */
/* Use nids from mount line: uml1,1@elan:uml2,2@elan:/lustre */
ptr = lsi->lsi_lmd->lmd_dev;
if (class_parse_nid(ptr, &nid, &ptr) == 0)
i++;
}
if (i == 0) {
CERROR("No valid MGS nids found.\n");
return -EINVAL;
}
mutex_lock(&mgc_start_lock);
len = strlen(LUSTRE_MGC_OBDNAME) + strlen(libcfs_nid2str(nid)) + 1;
OBD_ALLOC(mgcname, len);
OBD_ALLOC(niduuid, len + 2);
if (!mgcname || !niduuid)
GOTO(out_free, rc = -ENOMEM);
sprintf(mgcname, "%s%s", LUSTRE_MGC_OBDNAME, libcfs_nid2str(nid));
mgssec = lsi->lsi_lmd->lmd_mgssec ? lsi->lsi_lmd->lmd_mgssec : "";
OBD_ALLOC_PTR(data);
if (data == NULL)
GOTO(out_free, rc = -ENOMEM);
obd = class_name2obd(mgcname);
if (obd && !obd->obd_stopping) {
int recov_bk;
rc = obd_set_info_async(NULL, obd->obd_self_export,
strlen(KEY_MGSSEC), KEY_MGSSEC,
strlen(mgssec), mgssec, NULL);
if (rc)
GOTO(out_free, rc);
/* Re-using an existing MGC */
atomic_inc(&obd->u.cli.cl_mgc_refcount);
/* IR compatibility check, only for clients */
if (lmd_is_client(lsi->lsi_lmd)) {
int has_ir;
int vallen = sizeof(*data);
/* Start the MGS obd */
static int mgs_setup(struct obd_device *obd, struct lustre_cfg *lcfg)
{
struct lprocfs_static_vars lvars;
struct mgs_obd *mgs = &obd->u.mgs;
struct lustre_mount_info *lmi;
struct lustre_sb_info *lsi;
struct vfsmount *mnt;
int rc = 0;
ENTRY;
CDEBUG(D_CONFIG, "Starting MGS\n");
/* Find our disk */
lmi = server_get_mount(obd->obd_name);
if (!lmi)
RETURN(rc = -EINVAL);
mnt = lmi->lmi_mnt;
lsi = s2lsi(lmi->lmi_sb);
obd->obd_fsops = fsfilt_get_ops(MT_STR(lsi->lsi_ldd));
if (IS_ERR(obd->obd_fsops))
GOTO(err_put, rc = PTR_ERR(obd->obd_fsops));
if (lvfs_check_rdonly(lvfs_sbdev(mnt->mnt_sb))) {
CERROR("%s: Underlying device is marked as read-only. "
"Setup failed\n", obd->obd_name);
GOTO(err_ops, rc = -EROFS);
}
obd->u.obt.obt_magic = OBT_MAGIC;
obd->u.obt.obt_instance = 0;
/* namespace for mgs llog */
obd->obd_namespace = ldlm_namespace_new(obd ,"MGS",
LDLM_NAMESPACE_SERVER,
LDLM_NAMESPACE_MODEST,
LDLM_NS_TYPE_MGT);
if (obd->obd_namespace == NULL)
GOTO(err_ops, rc = -ENOMEM);
/* ldlm setup */
ptlrpc_init_client(LDLM_CB_REQUEST_PORTAL, LDLM_CB_REPLY_PORTAL,
"mgs_ldlm_client", &obd->obd_ldlm_client);
rc = mgs_fs_setup(obd, mnt);
if (rc) {
CERROR("%s: MGS filesystem method init failed: rc = %d\n",
obd->obd_name, rc);
GOTO(err_ns, rc);
}
rc = obd_llog_init(obd, &obd->obd_olg, obd, NULL);
if (rc)
GOTO(err_fs, rc);
/* No recovery for MGC's */
obd->obd_replayable = 0;
/* Internal mgs setup */
mgs_init_fsdb_list(obd);
cfs_mutex_init(&mgs->mgs_mutex);
mgs->mgs_start_time = cfs_time_current_sec();
/* Setup proc */
lprocfs_mgs_init_vars(&lvars);
if (lprocfs_obd_setup(obd, lvars.obd_vars) == 0) {
lproc_mgs_setup(obd);
rc = lprocfs_alloc_md_stats(obd, LPROC_MGS_LAST);
if (rc)
GOTO(err_llog, rc);
}
/* Start the service threads */
mgs->mgs_service =
ptlrpc_init_svc(MGS_NBUFS, MGS_BUFSIZE, MGS_MAXREQSIZE,
MGS_MAXREPSIZE, MGS_REQUEST_PORTAL,
MGC_REPLY_PORTAL, 2,
mgs_handle, LUSTRE_MGS_NAME,
obd->obd_proc_entry, target_print_req,
MGS_THREADS_AUTO_MIN, MGS_THREADS_AUTO_MAX,
"ll_mgs", LCT_MD_THREAD, NULL);
if (!mgs->mgs_service) {
CERROR("failed to start service\n");
GOTO(err_llog, rc = -ENOMEM);
}
rc = ptlrpc_start_threads(mgs->mgs_service);
if (rc)
GOTO(err_thread, rc);
ping_evictor_start();
CDEBUG(D_INFO, "MGS %s started\n", obd->obd_name);
RETURN(0);
err_thread:
ptlrpc_unregister_service(mgs->mgs_service);
err_llog:
lproc_mgs_cleanup(obd);
obd_llog_finish(obd, 0);
err_fs:
/* No extra cleanup needed for llog_init_commit_thread() */
mgs_fs_cleanup(obd);
err_ns:
ldlm_namespace_free(obd->obd_namespace, NULL, 0);
obd->obd_namespace = NULL;
err_ops:
fsfilt_put_ops(obd->obd_fsops);
err_put:
server_put_mount(obd->obd_name, mnt);
mgs->mgs_sb = 0;
return rc;
}