/** Set up a mgc obd to process startup logs
*
* \param sb [in] super block of the mgc obd
*
* \retval 0 success, otherwise error code
*/
int lustre_start_mgc(struct super_block *sb)
{
struct obd_connect_data *data = NULL;
struct lustre_sb_info *lsi = s2lsi(sb);
struct obd_device *obd;
struct obd_export *exp;
struct obd_uuid *uuid;
class_uuid_t uuidc;
lnet_nid_t nid;
char *mgcname = NULL, *niduuid = NULL, *mgssec = NULL;
char *ptr;
int recov_bk;
int rc = 0, i = 0, j, len;
LASSERT(lsi->lsi_lmd);
/* Find the first non-lo MGS nid for our MGC name */
if (IS_SERVER(lsi)) {
/* mount -o mgsnode=nid */
ptr = lsi->lsi_lmd->lmd_mgs;
if (lsi->lsi_lmd->lmd_mgs &&
(class_parse_nid(lsi->lsi_lmd->lmd_mgs, &nid, &ptr) == 0)) {
i++;
} else if (IS_MGS(lsi)) {
lnet_process_id_t id;
while ((rc = LNetGetId(i++, &id)) != -ENOENT) {
if (LNET_NETTYP(LNET_NIDNET(id.nid)) == LOLND)
continue;
nid = id.nid;
i++;
break;
}
}
} else { /* client */
/* Use nids from mount line: uml1,1@elan:uml2,2@elan:/lustre */
ptr = lsi->lsi_lmd->lmd_dev;
if (class_parse_nid(ptr, &nid, &ptr) == 0)
i++;
}
if (i == 0) {
CERROR("No valid MGS nids found.\n");
return -EINVAL;
}
mutex_lock(&mgc_start_lock);
len = strlen(LUSTRE_MGC_OBDNAME) + strlen(libcfs_nid2str(nid)) + 1;
OBD_ALLOC(mgcname, len);
OBD_ALLOC(niduuid, len + 2);
if (!mgcname || !niduuid)
GOTO(out_free, rc = -ENOMEM);
sprintf(mgcname, "%s%s", LUSTRE_MGC_OBDNAME, libcfs_nid2str(nid));
mgssec = lsi->lsi_lmd->lmd_mgssec ? lsi->lsi_lmd->lmd_mgssec : "";
OBD_ALLOC_PTR(data);
if (data == NULL)
GOTO(out_free, rc = -ENOMEM);
obd = class_name2obd(mgcname);
if (obd && !obd->obd_stopping) {
rc = obd_set_info_async(NULL, obd->obd_self_export,
strlen(KEY_MGSSEC), KEY_MGSSEC,
strlen(mgssec), mgssec, NULL);
if (rc)
GOTO(out_free, rc);
/* Re-using an existing MGC */
atomic_inc(&obd->u.cli.cl_mgc_refcount);
/* IR compatibility check, only for clients */
if (lmd_is_client(lsi->lsi_lmd)) {
int has_ir;
int vallen = sizeof(*data);
__u32 *flags = &lsi->lsi_lmd->lmd_flags;
rc = obd_get_info(NULL, obd->obd_self_export,
strlen(KEY_CONN_DATA), KEY_CONN_DATA,
&vallen, data, NULL);
LASSERT(rc == 0);
has_ir = OCD_HAS_FLAG(data, IMP_RECOV);
if (has_ir ^ !(*flags & LMD_FLG_NOIR)) {
/* LMD_FLG_NOIR is for test purpose only */
LCONSOLE_WARN(
"Trying to mount a client with IR setting "
"not compatible with current mgc. "
"Force to use current mgc setting that is "
"IR %s.\n",
has_ir ? "enabled" : "disabled");
if (has_ir)
*flags &= ~LMD_FLG_NOIR;
else
*flags |= LMD_FLG_NOIR;
}
}
recov_bk = 0;
/* If we are restarting the MGS, don't try to keep the MGC's
old connection, or registration will fail. */
if (IS_MGS(lsi)) {
CDEBUG(D_MOUNT, "New MGS with live MGC\n");
recov_bk = 1;
}
/* Try all connections, but only once (again).
We don't want to block another target from starting
(using its local copy of the log), but we do want to connect
if at all possible. */
recov_bk++;
CDEBUG(D_MOUNT, "%s: Set MGC reconnect %d\n", mgcname,recov_bk);
rc = obd_set_info_async(NULL, obd->obd_self_export,
sizeof(KEY_INIT_RECOV_BACKUP),
KEY_INIT_RECOV_BACKUP,
sizeof(recov_bk), &recov_bk, NULL);
GOTO(out, rc = 0);
}
CDEBUG(D_MOUNT, "Start MGC '%s'\n", mgcname);
/* Add the primary nids for the MGS */
i = 0;
sprintf(niduuid, "%s_%x", mgcname, i);
if (IS_SERVER(lsi)) {
ptr = lsi->lsi_lmd->lmd_mgs;
if (IS_MGS(lsi)) {
/* Use local nids (including LO) */
lnet_process_id_t id;
while ((rc = LNetGetId(i++, &id)) != -ENOENT) {
rc = do_lcfg(mgcname, id.nid,
LCFG_ADD_UUID, niduuid, 0,0,0);
}
} else {
/* Use mgsnode= nids */
/* mount -o mgsnode=nid */
if (lsi->lsi_lmd->lmd_mgs) {
ptr = lsi->lsi_lmd->lmd_mgs;
} else if (class_find_param(ptr, PARAM_MGSNODE,
&ptr) != 0) {
CERROR("No MGS nids given.\n");
GOTO(out_free, rc = -EINVAL);
}
while (class_parse_nid(ptr, &nid, &ptr) == 0) {
rc = do_lcfg(mgcname, nid,
LCFG_ADD_UUID, niduuid, 0,0,0);
i++;
}
}
} else { /* client */
/* Use nids from mount line: uml1,1@elan:uml2,2@elan:/lustre */
ptr = lsi->lsi_lmd->lmd_dev;
while (class_parse_nid(ptr, &nid, &ptr) == 0) {
rc = do_lcfg(mgcname, nid,
LCFG_ADD_UUID, niduuid, 0,0,0);
i++;
/* Stop at the first failover nid */
if (*ptr == ':')
break;
}
}
if (i == 0) {
CERROR("No valid MGS nids found.\n");
GOTO(out_free, rc = -EINVAL);
}
lsi->lsi_lmd->lmd_mgs_failnodes = 1;
/* Random uuid for MGC allows easier reconnects */
OBD_ALLOC_PTR(uuid);
ll_generate_random_uuid(uuidc);
class_uuid_unparse(uuidc, uuid);
/* Start the MGC */
rc = lustre_start_simple(mgcname, LUSTRE_MGC_NAME,
(char *)uuid->uuid, LUSTRE_MGS_OBDNAME,
niduuid, 0, 0);
OBD_FREE_PTR(uuid);
if (rc)
GOTO(out_free, rc);
/* Add any failover MGS nids */
i = 1;
while (ptr && ((*ptr == ':' ||
class_find_param(ptr, PARAM_MGSNODE, &ptr) == 0))) {
/* New failover node */
sprintf(niduuid, "%s_%x", mgcname, i);
j = 0;
while (class_parse_nid_quiet(ptr, &nid, &ptr) == 0) {
j++;
rc = do_lcfg(mgcname, nid,
LCFG_ADD_UUID, niduuid, 0,0,0);
if (*ptr == ':')
break;
}
if (j > 0) {
rc = do_lcfg(mgcname, 0, LCFG_ADD_CONN,
niduuid, 0, 0, 0);
i++;
} else {
/* at ":/fsname" */
break;
}
}
lsi->lsi_lmd->lmd_mgs_failnodes = i;
obd = class_name2obd(mgcname);
if (!obd) {
CERROR("Can't find mgcobd %s\n", mgcname);
GOTO(out_free, rc = -ENOTCONN);
}
rc = obd_set_info_async(NULL, obd->obd_self_export,
strlen(KEY_MGSSEC), KEY_MGSSEC,
strlen(mgssec), mgssec, NULL);
if (rc)
GOTO(out_free, rc);
/* Keep a refcount of servers/clients who started with "mount",
so we know when we can get rid of the mgc. */
atomic_set(&obd->u.cli.cl_mgc_refcount, 1);
/* Try all connections, but only once. */
recov_bk = 1;
rc = obd_set_info_async(NULL, obd->obd_self_export,
sizeof(KEY_INIT_RECOV_BACKUP),
KEY_INIT_RECOV_BACKUP,
sizeof(recov_bk), &recov_bk, NULL);
if (rc)
/* nonfatal */
CWARN("can't set %s %d\n", KEY_INIT_RECOV_BACKUP, rc);
/* We connect to the MGS at setup, and don't disconnect until cleanup */
data->ocd_connect_flags = OBD_CONNECT_VERSION | OBD_CONNECT_AT |
OBD_CONNECT_FULL20 | OBD_CONNECT_IMP_RECOV |
OBD_CONNECT_LVB_TYPE;
#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(3, 2, 50, 0)
data->ocd_connect_flags |= OBD_CONNECT_MNE_SWAB;
#else
#warning "LU-1644: Remove old OBD_CONNECT_MNE_SWAB fixup and imp_need_mne_swab"
#endif
if (lmd_is_client(lsi->lsi_lmd) &&
lsi->lsi_lmd->lmd_flags & LMD_FLG_NOIR)
data->ocd_connect_flags &= ~OBD_CONNECT_IMP_RECOV;
data->ocd_version = LUSTRE_VERSION_CODE;
rc = obd_connect(NULL, &exp, obd, &(obd->obd_uuid), data, NULL);
if (rc) {
CERROR("connect failed %d\n", rc);
GOTO(out, rc);
}
obd->u.cli.cl_mgc_mgsexp = exp;
out:
/* Keep the mgc info in the sb. Note that many lsi's can point
to the same mgc.*/
lsi->lsi_mgc = obd;
out_free:
mutex_unlock(&mgc_start_lock);
if (data)
OBD_FREE_PTR(data);
if (mgcname)
OBD_FREE(mgcname, len);
if (niduuid)
OBD_FREE(niduuid, len + 2);
return rc;
}
/** Set up a mgc obd to process startup logs
*
* \param sb [in] super block of the mgc obd
*
* \retval 0 success, otherwise error code
*/
int lustre_start_mgc(struct super_block *sb)
{
struct obd_connect_data *data = NULL;
struct lustre_sb_info *lsi = s2lsi(sb);
struct obd_device *obd;
struct obd_export *exp;
struct obd_uuid *uuid;
class_uuid_t uuidc;
lnet_nid_t nid;
char nidstr[LNET_NIDSTR_SIZE];
char *mgcname = NULL, *niduuid = NULL, *mgssec = NULL;
char *ptr;
int rc = 0, i = 0, j;
LASSERT(lsi->lsi_lmd);
/* Use nids from mount line: uml1,1@elan:uml2,2@elan:/lustre */
ptr = lsi->lsi_lmd->lmd_dev;
if (class_parse_nid(ptr, &nid, &ptr) == 0)
i++;
if (i == 0) {
CERROR("No valid MGS nids found.\n");
return -EINVAL;
}
mutex_lock(&mgc_start_lock);
libcfs_nid2str_r(nid, nidstr, sizeof(nidstr));
mgcname = kasprintf(GFP_NOFS,
"%s%s", LUSTRE_MGC_OBDNAME, nidstr);
niduuid = kasprintf(GFP_NOFS, "%s_%x", mgcname, i);
if (!mgcname || !niduuid) {
rc = -ENOMEM;
goto out_free;
}
mgssec = lsi->lsi_lmd->lmd_mgssec ? lsi->lsi_lmd->lmd_mgssec : "";
data = kzalloc(sizeof(*data), GFP_NOFS);
if (!data) {
rc = -ENOMEM;
goto out_free;
}
obd = class_name2obd(mgcname);
if (obd && !obd->obd_stopping) {
int recov_bk;
rc = obd_set_info_async(NULL, obd->obd_self_export,
strlen(KEY_MGSSEC), KEY_MGSSEC,
strlen(mgssec), mgssec, NULL);
if (rc)
goto out_free;
/* Re-using an existing MGC */
atomic_inc(&obd->u.cli.cl_mgc_refcount);
/* IR compatibility check, only for clients */
if (lmd_is_client(lsi->lsi_lmd)) {
int has_ir;
int vallen = sizeof(*data);
__u32 *flags = &lsi->lsi_lmd->lmd_flags;
rc = obd_get_info(NULL, obd->obd_self_export,
strlen(KEY_CONN_DATA), KEY_CONN_DATA,
&vallen, data, NULL);
LASSERT(rc == 0);
has_ir = OCD_HAS_FLAG(data, IMP_RECOV);
if (has_ir ^ !(*flags & LMD_FLG_NOIR)) {
/* LMD_FLG_NOIR is for test purpose only */
LCONSOLE_WARN(
"Trying to mount a client with IR setting not compatible with current mgc. Force to use current mgc setting that is IR %s.\n",
has_ir ? "enabled" : "disabled");
if (has_ir)
*flags &= ~LMD_FLG_NOIR;
else
*flags |= LMD_FLG_NOIR;
}
}
recov_bk = 0;
/* Try all connections, but only once (again).
We don't want to block another target from starting
(using its local copy of the log), but we do want to connect
if at all possible. */
recov_bk++;
CDEBUG(D_MOUNT, "%s: Set MGC reconnect %d\n", mgcname,
recov_bk);
rc = obd_set_info_async(NULL, obd->obd_self_export,
sizeof(KEY_INIT_RECOV_BACKUP),
KEY_INIT_RECOV_BACKUP,
sizeof(recov_bk), &recov_bk, NULL);
rc = 0;
goto out;
}
CDEBUG(D_MOUNT, "Start MGC '%s'\n", mgcname);
/* Add the primary nids for the MGS */
i = 0;
/* Use nids from mount line: uml1,1@elan:uml2,2@elan:/lustre */
ptr = lsi->lsi_lmd->lmd_dev;
while (class_parse_nid(ptr, &nid, &ptr) == 0) {
rc = do_lcfg(mgcname, nid,
LCFG_ADD_UUID, niduuid, NULL, NULL, NULL);
i++;
/* Stop at the first failover nid */
if (*ptr == ':')
break;
}
if (i == 0) {
CERROR("No valid MGS nids found.\n");
rc = -EINVAL;
goto out_free;
}
lsi->lsi_lmd->lmd_mgs_failnodes = 1;
/* Random uuid for MGC allows easier reconnects */
uuid = kzalloc(sizeof(*uuid), GFP_NOFS);
if (!uuid) {
rc = -ENOMEM;
goto out_free;
}
ll_generate_random_uuid(uuidc);
class_uuid_unparse(uuidc, uuid);
/* Start the MGC */
rc = lustre_start_simple(mgcname, LUSTRE_MGC_NAME,
(char *)uuid->uuid, LUSTRE_MGS_OBDNAME,
niduuid, NULL, NULL);
kfree(uuid);
if (rc)
goto out_free;
/* Add any failover MGS nids */
i = 1;
while (ptr && ((*ptr == ':' ||
class_find_param(ptr, PARAM_MGSNODE, &ptr) == 0))) {
/* New failover node */
sprintf(niduuid, "%s_%x", mgcname, i);
j = 0;
while (class_parse_nid_quiet(ptr, &nid, &ptr) == 0) {
j++;
rc = do_lcfg(mgcname, nid,
LCFG_ADD_UUID, niduuid, NULL, NULL, NULL);
if (*ptr == ':')
break;
}
if (j > 0) {
rc = do_lcfg(mgcname, 0, LCFG_ADD_CONN,
niduuid, NULL, NULL, NULL);
i++;
} else {
/* at ":/fsname" */
break;
}
}
lsi->lsi_lmd->lmd_mgs_failnodes = i;
obd = class_name2obd(mgcname);
if (!obd) {
CERROR("Can't find mgcobd %s\n", mgcname);
rc = -ENOTCONN;
goto out_free;
}
rc = obd_set_info_async(NULL, obd->obd_self_export,
strlen(KEY_MGSSEC), KEY_MGSSEC,
strlen(mgssec), mgssec, NULL);
if (rc)
goto out_free;
/* Keep a refcount of servers/clients who started with "mount",
so we know when we can get rid of the mgc. */
atomic_set(&obd->u.cli.cl_mgc_refcount, 1);
/* We connect to the MGS at setup, and don't disconnect until cleanup */
data->ocd_connect_flags = OBD_CONNECT_VERSION | OBD_CONNECT_AT |
OBD_CONNECT_FULL20 | OBD_CONNECT_IMP_RECOV |
OBD_CONNECT_LVB_TYPE;
#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(3, 2, 50, 0)
data->ocd_connect_flags |= OBD_CONNECT_MNE_SWAB;
#else
#warning "LU-1644: Remove old OBD_CONNECT_MNE_SWAB fixup and imp_need_mne_swab"
#endif
if (lmd_is_client(lsi->lsi_lmd) &&
lsi->lsi_lmd->lmd_flags & LMD_FLG_NOIR)
data->ocd_connect_flags &= ~OBD_CONNECT_IMP_RECOV;
data->ocd_version = LUSTRE_VERSION_CODE;
rc = obd_connect(NULL, &exp, obd, &(obd->obd_uuid), data, NULL);
if (rc) {
CERROR("connect failed %d\n", rc);
goto out;
}
obd->u.cli.cl_mgc_mgsexp = exp;
out:
/* Keep the mgc info in the sb. Note that many lsi's can point
to the same mgc.*/
lsi->lsi_mgc = obd;
out_free:
mutex_unlock(&mgc_start_lock);
kfree(data);
kfree(mgcname);
kfree(niduuid);
return rc;
}