int hfi_context_open(int unit, int port, uint64_t open_timeout)
{
int fd;
char dev_name[MAXPATHLEN];
if (unit != HFI_UNIT_ID_ANY && unit >= 0)
snprintf(dev_name, sizeof(dev_name), "%s_%u", HFI_DEVICE_PATH,
unit);
else
snprintf(dev_name, sizeof(dev_name), "%s", HFI_DEVICE_PATH);
if (hfi_wait_for_device(dev_name, (long)open_timeout) == -1) {
_HFI_DBG("Could not find an HFI Unit on device "
"%s (%lds elapsed)", dev_name,
(long)open_timeout / 1000);
return -1;
}
if ((fd = open(dev_name, O_RDWR)) == -1) {
_HFI_DBG("(host:Can't open %s for reading and writing",
dev_name);
return -1;
}
if (fcntl(fd, F_SETFD, FD_CLOEXEC))
_HFI_INFO("Failed to set close on exec for device: %s\n",
strerror(errno));
return fd;
}
int hfi_context_open(int unit, int port, uint64_t open_timeout)
{
int fd;
char dev_name[MAXPATHLEN];
if (unit != HFI_UNIT_ID_ANY && unit >= 0)
snprintf(dev_name, sizeof(dev_name), "%s_%u", HFI_DEVICE_PATH,
unit);
else
snprintf(dev_name, sizeof(dev_name), "%s", HFI_DEVICE_PATH);
if (hfi_wait_for_device(dev_name, (long)open_timeout) == -1) {
_HFI_DBG("Could not find an HFI Unit on device "
"%s (%lds elapsed)", dev_name,
(long)open_timeout / 1000);
return -1;
}
#ifdef PSM2_SUPPORT_IW_CMD_API
{
int major = -1;
struct stat sbuf;
if ((!hfi_sysfs_stat("user_sw_version", &sbuf)) &&
S_ISREG(sbuf.st_mode))
{
int64_t user_major_minor;
if (hfi_sysfs_read_s64("user_sw_version", &user_major_minor, 0) >= 0)
{
/* unpack the major version stored in user_major_minor into major:; */
major = (user_major_minor >> HFI1_SWMAJOR_SHIFT) &
((1 << HFI1_SWMAJOR_SHIFT)-1);
}
}
void hfi_flush_egr_bufs(struct _hfi_ctrl *ctrl)
{
uint64_t head = __le64_to_cpu(*ctrl->__hfi_rcvegrhead);
uint64_t tail = __le64_to_cpu(*ctrl->__hfi_rcvegrtail);
if ((head % ctrl->__hfi_tidegrcnt) ==
((tail + 1) % ctrl->__hfi_tidegrcnt)) {
_HFI_DBG
("eager array full after overflow, flushing (head %llx, tail %llx\n",
(long long)head, (long long)tail);
*ctrl->__hfi_rcvegrhead = __cpu_to_le64(tail);
}
}
/* Given the unit number, return an error, or the corresponding GID
For now, it's used only so the MPI code can determine its fabric ID.
Returns an int, so -1 indicates an error.
No error print because we call this for both potential
ports without knowing if both ports exist (or are connected) */
int hfi_get_port_gid(int unit, int port, uint64_t *hi, uint64_t *lo)
{
int ret;
char *gid_str = NULL;
ret = hfi_sysfs_port_read(unit, port, "gids/0", &gid_str);
if (ret == -1) {
if (errno == ENODEV)
/* this is "normal" for port != 1, on single
* port chips */
_HFI_VDBG("Failed to get GID for unit %u:%u: %s\n",
unit, port, strerror(errno));
else
_HFI_DBG("Failed to get GID for unit %u:%u: %s\n",
unit, port, strerror(errno));
} else {
int gid[8];
if (sscanf(gid_str, "%4x:%4x:%4x:%4x:%4x:%4x:%4x:%4x",
&gid[0], &gid[1], &gid[2], &gid[3],
&gid[4], &gid[5], &gid[6], &gid[7]) != 8) {
_HFI_DBG("Failed to parse GID for unit %u:%u: %s\n",
unit, port, gid_str);
ret = -1;
} else {
*hi = (((uint64_t) gid[0]) << 48) | (((uint64_t) gid[1])
<< 32) |
(((uint64_t)
gid[2]) << 16) | (((uint64_t) gid[3]) << 0);
*lo = (((uint64_t) gid[4]) << 48) | (((uint64_t) gid[5])
<< 32) |
(((uint64_t)
gid[6]) << 16) | (((uint64_t) gid[7]) << 0);
}
free(gid_str);
}
return ret;
}
/* ack event bits, and clear them. Usage is check *spi_sendbuf_status,
pass bits you are prepared to handle to hfi_event_ack(), perform the
appropriate actions for bits that were set, and then (if appropriate)
check the bits again. */
int hfi_event_ack(struct _hfi_ctrl *ctrl, __u64 ackbits)
{
struct hfi1_cmd cmd;
cmd.type = HFI1_CMD_ACK_EVENT;
cmd.len = 0;
cmd.addr = ackbits;
if (hfi_cmd_write(ctrl->fd, &cmd, sizeof(cmd)) == -1) {
if (errno != EINVAL) /* not implemented in driver. */
_HFI_DBG("event ack failed: %s\n", strerror(errno));
return -1;
}
return 0;
}
/* Returns an int, so -1 indicates an error. */
int hfi_get_port_index2pkey(int unit, int port, int index)
{
int ret;
int64_t val;
char index2pkeypath[16];
snprintf(index2pkeypath, sizeof(index2pkeypath), "pkeys/%d", index);
ret = hfi_sysfs_port_read_s64(unit, port, index2pkeypath, &val, 0);
if (ret == -1) {
_HFI_DBG
("Failed to get index2pkey mapping for index %d unit %u:%u: %s\n",
index, unit, port, strerror(errno));
} else
ret = val;
return ret;
}
/* Returns an int, so -1 indicates an error. */
int hfi_get_port_vl2mtu(int unit, int port, int vl)
{
int ret;
int64_t val;
char vl2mtupath[16];
snprintf(vl2mtupath, sizeof(vl2mtupath), "vl2mtu/%d", vl);
ret = hfi_sysfs_port_read_s64(unit, port, vl2mtupath, &val, 0);
if (ret == -1) {
_HFI_DBG
("Failed to get VL2MTU mapping for VL %d unit %u:%u: %s\n",
vl, unit, port, strerror(errno));
} else
ret = val;
return ret;
}
/* Returns an int, so -1 indicates an error. */
int hfi_get_port_sc2vl(int unit, int port, int sc)
{
int ret;
int64_t val;
char sc2vlpath[16];
snprintf(sc2vlpath, sizeof(sc2vlpath), "sc2vl/%d", sc);
ret = hfi_sysfs_port_read_s64(unit, port, sc2vlpath, &val, 0);
if (ret == -1) {
_HFI_DBG
("Failed to get SC2VL mapping for SC %d unit %u:%u: %s\n",
sc, unit, port, strerror(errno));
} else
ret = val;
return ret;
}
/* Returns an int, so -1 indicates an error. */
int hfi_get_port_sl2sc(int unit, int port, int sl)
{
int ret;
int64_t val;
char sl2scpath[16];
snprintf(sl2scpath, sizeof(sl2scpath), "sl2sc/%d", sl);
ret = hfi_sysfs_port_read_s64(unit, port, sl2scpath, &val, 0);
if (ret == -1) {
_HFI_DBG
("Failed to get SL2SC mapping for SL %d unit %u:%u: %s\n",
sl, unit, port, strerror(errno));
} else
ret = val;
return ret;
}
static
psm_error_t
psmi_ep_open_device(const psm_ep_t ep,
const struct psm_ep_open_opts *opts,
const psm_uuid_t unique_job_key,
struct psmi_context *context, psm_epid_t *epid)
{
psm_error_t err = PSM_OK;
/* Skip affinity. No affinity if:
* 1. User explicitly sets no-affinity=YES in environment.
* 2. User doesn't set affinity in environment and PSM is opened with
* option affinity skip.
*/
if (psmi_ep_device_is_enabled(ep, PTL_DEVID_IPS)) {
uint32_t rcvthread_flags;
union psmi_envvar_val env_rcvthread;
static int norcvthread; /* only for first rail */
ep->out_sl = opts->outsl;
if ((err =
psmi_context_open(ep, opts->unit, opts->port,
unique_job_key, opts->timeout,
context)) != PSM_OK)
goto fail;
_HFI_DBG("[%d]use unit %d port %d\n", getpid(),
context->ctrl->__hfi_unit, 1);
/* At this point, we have the unit id and port number, so
* check if pkey is not 0x0/0x7fff/0xffff, and match one
* of the pkey in table.
*/
if ((err =
psmi_ep_verify_pkey(ep, (uint16_t) opts->network_pkey,
&ep->network_pkey)) != PSM_OK)
goto fail;
/* See if we want to activate support for receive thread */
psmi_getenv("PSM_RCVTHREAD",
"Recv thread flags (0 disables thread)",
PSMI_ENVVAR_LEVEL_USER, PSMI_ENVVAR_TYPE_UINT_FLAGS,
(union psmi_envvar_val)(norcvthread++ ? 0 :
PSMI_RCVTHREAD_FLAGS),
&env_rcvthread);
rcvthread_flags = env_rcvthread.e_uint;
/* If enabled, use the pollurg capability to implement a receive
* interrupt thread that can handle urg packets */
if (rcvthread_flags) {
context->runtime_flags |= PSMI_RUNTIME_RCVTHREAD;
#ifdef PSMI_PLOCK_IS_NOLOCK
psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR,
"#define PSMI_PLOCK_IS_NOLOCK not functional yet "
"with RCVTHREAD on");
#endif
}
context->rcvthread_flags = rcvthread_flags;
*epid = context->epid;
} else if (psmi_ep_device_is_enabled(ep, PTL_DEVID_AMSH)) {
int rank;
/* In shm-only mode, we need to derive a valid epid
* based on our rank. We try to get it from the
* environment if its available, or resort to using
* our PID as the rank.
*/
union psmi_envvar_val env_rankid;
if (psmi_getenv
("MPI_LOCALRANKID", "Shared context rankid",
PSMI_ENVVAR_LEVEL_HIDDEN, PSMI_ENVVAR_TYPE_INT,
(union psmi_envvar_val)-1, &env_rankid)) {
if (psmi_getenv
("PSC_MPI_NODE_RANK",
"Shared context rankid",
PSMI_ENVVAR_LEVEL_HIDDEN,
PSMI_ENVVAR_TYPE_INT,
(union psmi_envvar_val)-1, &env_rankid)) {
rank = getpid();
} else
rank = env_rankid.e_int;
} else
rank = env_rankid.e_int;
/*
* We use a LID of 0 for non-HFI communication.
* Since a jobkey is not available from IPS, pull the
* first 16 bits from the UUID.
*/
*epid = PSMI_EPID_PACK(((uint16_t *) unique_job_key)[0],
(rank >> 3), rank, 0,
PSMI_HFI_TYPE_DEFAULT, 0x7, rank);
} else {
/*
* This function is necessary in a udev-based world. There can be an
* arbitrarily long (but typically less than one second) delay between
* a driver getting loaded and any dynamic special files turning up.
*
* The timeout is in milliseconds. A value of zero means "callee
* decides timeout". Negative is infinite.
*
* Returns 0 on success, -1 on error or timeout. Check errno to see
* whether there was a timeout (ETIMEDOUT) or an error (any other
* non-zero value).
*/
int hfi_wait_for_device(const char *path, long timeout)
{
int saved_errno;
struct stat st;
long elapsed;
int ret;
if (timeout == 0)
timeout = 15000;
elapsed = 0;
while (1) {
static const long default_ms = 250;
struct timespec req = { 0 };
long ms;
ret = stat(path, &st);
saved_errno = errno;
if (ret == 0 || (ret == -1 && errno != ENOENT))
break;
if (timeout - elapsed == 0) {
saved_errno = ETIMEDOUT;
break;
}
if (elapsed == 0) {
if (timeout == -1)
_HFI_DBG
("Device file %s not present on first check; "
"waiting indefinitely...\n", path);
else
_HFI_DBG
("Device file %s not present on first check; "
"waiting up to %.1f seconds...\n", path,
timeout / 1e3);
}
if (timeout < 0 || timeout - elapsed >= default_ms)
ms = default_ms;
else
ms = timeout;
elapsed += ms;
req.tv_nsec = ms * 1000000;
ret = nanosleep(&req, NULL);
saved_errno = errno;
if (ret == -1)
break;
}
if (ret == 0)
_HFI_DBG("Found %s after %.1f seconds\n", path, elapsed / 1e3);
else
_HFI_INFO
("The %s device failed to appear after %.1f seconds: %s\n",
path, elapsed / 1e3, strerror(saved_errno));
errno = saved_errno;
return ret;
}
/* Given the unit number, return an error, or the corresponding LID
For now, it's used only so the MPI code can determine it's own
LID, and which other LIDs (if any) are also assigned to this node
Returns an int, so -1 indicates an error. 0 may indicate that
the unit is valid, but no LID has been assigned.
No error print because we call this for both potential
ports without knowing if both ports exist (or are connected) */
int hfi_get_port_lid(int unit, int port)
{
int ret;
char *state;
int64_t val;
ret = hfi_sysfs_port_read(unit, port, "phys_state", &state);
if (ret == -1) {
if (errno == ENODEV)
/* this is "normal" for port != 1, on single port chips */
_HFI_VDBG
("Failed to get phys_state for unit %u:%u: %s\n",
unit, port, strerror(errno));
else
_HFI_DBG
("Failed to get phys_state for unit %u:%u: %s\n",
unit, port, strerror(errno));
} else {
if (strncmp(state, "5: LinkUp", 9)) {
_HFI_DBG("Link is not Up for unit %u:%u\n", unit, port);
ret = -1;
}
free(state);
}
/* If link is not up, we think lid not valid */
if (ret == -1)
return ret;
ret = hfi_sysfs_port_read_s64(unit, port, "lid", &val, 0);
_HFI_VDBG("hfi_get_port_lid: ret %d, unit %d port %d\n", ret, unit,
port);
if (ret == -1) {
if (errno == ENODEV)
/* this is "normal" for port != 1, on single port chips */
_HFI_VDBG("Failed to get LID for unit %u:%u: %s\n",
unit, port, strerror(errno));
else
_HFI_DBG("Failed to get LID for unit %u:%u: %s\n",
unit, port, strerror(errno));
} else {
ret = val;
/* disable this feature since we don't have a way to provide
file descriptor in multiple context case. */
#if 0
if (getenv("HFI_DIAG_LID_LOOP")) {
/* provides diagnostic ability to run MPI, etc. even */
/* on loopback, by claiming a different LID for each context */
struct hfi1_ctxt_info info;
struct hfi1_cmd cmd;
cmd.type = HFI1_CMD_CTXT_INFO;
cmd.cmd.ctxt_info = (uintptr_t) &info;
if (__hfi_lastfd == -1)
_HFI_INFO
("Can't run CONTEXT_INFO for lid_loop, fd not set\n");
else if (write(__hfi_lastfd, &cmd, sizeof(cmd)) == -1)
_HFI_INFO("CONTEXT_INFO command failed: %s\n",
strerror(errno));
else if (!info.context)
_HFI_INFO("CONTEXT_INFO returned context 0!\n");
else {
_HFI_PRDBG
("Using lid 0x%x, base %x, context %x\n",
ret + info.context, ret, info.context);
ret += info.context;
}
}
#endif
}
return ret;
}
