static void ft_show_test_info(void)
{
printf("[%s] ", test_info.prov_name);
switch (test_info.test_type) {
case FT_TEST_LATENCY:
printf("latency");
break;
case FT_TEST_BANDWIDTH:
printf("bandwidth");
break;
default:
break;
}
printf( " %s", fi_tostr(&test_info.ep_type, FI_TYPE_EP_TYPE));
printf( " [%s]", fi_tostr(&test_info.caps, FI_TYPE_CAPS));
switch (test_info.class_function) {
case FT_FUNC_SEND:
printf(" send");
break;
case FT_FUNC_SENDV:
printf(" sendv");
break;
case FT_FUNC_SENDMSG:
printf(" sendmsg");
break;
default:
break;
}
printf("\n");
}
int print_short_info(struct fi_info *info) {
for (struct fi_info *cur = info; cur; cur = cur->next) {
printf("%s: %s\n", cur->fabric_attr->prov_name, cur->fabric_attr->name);
printf(" version: %d.%d\n", FI_MAJOR(cur->fabric_attr->prov_version), FI_MINOR(cur->fabric_attr->prov_version));
printf(" type: %s\n", fi_tostr(&cur->ep_attr->type, FI_TYPE_EP_TYPE));
printf(" protocol: %s\n", fi_tostr(&cur->ep_attr->protocol, FI_TYPE_PROTOCOL));
}
return EXIT_SUCCESS;
}
static int psmx2_domain_check_features(struct psmx2_fid_domain *domain,
uint64_t ep_cap)
{
if ((domain->caps & ep_cap & ~PSMX2_SUB_CAPS) !=
(ep_cap & ~PSMX2_SUB_CAPS)) {
uint64_t mask = ~PSMX2_SUB_CAPS;
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"caps mismatch: domain->caps=%s,\n ep->caps=%s,\n mask=%s\n",
fi_tostr(&domain->caps, FI_TYPE_CAPS),
fi_tostr(&ep_cap, FI_TYPE_CAPS),
fi_tostr(&mask, FI_TYPE_CAPS));
return -FI_EOPNOTSUPP;
}
return 0;
}
int print_long_info(struct fi_info *info) {
for (struct fi_info *cur = info; cur; cur = cur->next) {
printf("---\n");
printf("%s", fi_tostr(cur, FI_TYPE_INFO));
}
return EXIT_SUCCESS;
}
static int
av_test_open_close(enum fi_av_type type, int count, uint64_t flags)
{
int ret;
struct fi_av_attr attr;
struct fid_av *av;
memset(&attr, 0, sizeof(attr));
attr.type = type;
attr.count = count;
attr.flags = flags;
ret = fi_av_open(domain, &attr, &av, NULL);
if (ret != 0) {
sprintf(err_buf, "fi_av_open(%d, %s) = %d, %s",
count, fi_tostr(&type, FI_TYPE_AV_TYPE),
ret, fi_strerror(-ret));
return ret;
}
ret = fi_close(&av->fid);
if (ret != 0) {
sprintf(err_buf, "close(av) = %d, %s", ret, fi_strerror(-ret));
return ret;
}
return 0;
}
static int print_short_info(struct fi_info *info)
{
struct fi_info *cur;
for (cur = info; cur; cur = cur->next) {
printf("provider: %s\n", cur->fabric_attr->prov_name);
printf(" fabric: %s\n", cur->fabric_attr->name),
printf(" domain: %s\n", cur->domain_attr->name),
printf(" version: %d.%d\n", FI_MAJOR(cur->fabric_attr->prov_version),
FI_MINOR(cur->fabric_attr->prov_version));
if (!list_providers) {
printf(" type: %s\n", fi_tostr(&cur->ep_attr->type, FI_TYPE_EP_TYPE));
printf(" protocol: %s\n", fi_tostr(&cur->ep_attr->protocol, FI_TYPE_PROTOCOL));
}
}
return EXIT_SUCCESS;
}
static int read_allowed(uint64_t rma_amo, uint64_t caps, uint64_t rcaps)
{
dbg_printf("read %s caps:%s, rcaps:%s\n",
fi_tostr(&rma_amo, FI_TYPE_CAPS),
fi_tostr(&caps, FI_TYPE_CAPS),
fi_tostr(&rcaps, FI_TYPE_CAPS));
if ((caps & rma_amo) &&
((caps & FI_READ) || !(caps & FI_WRITE))) {
if ((rcaps & rma_amo) &&
((rcaps & FI_REMOTE_READ) ||
(!(rcaps & (FI_READ | FI_WRITE | FI_REMOTE_WRITE)))
)
) {
return 1;
}
}
return 0;
}
static int ft_fw_server(void)
{
struct fi_info *hints, *info;
int ret;
hints = fi_allocinfo();
if (!hints)
return -FI_ENOMEM;
do {
ret = ft_fw_recv(sock, &test_info, sizeof test_info);
if (ret) {
if (ret == -FI_ENOTCONN)
ret = 0;
break;
}
ft_fw_convert_info(hints, &test_info);
printf("Starting test %d-%d: ", test_info.test_index,
test_info.test_subindex);
ft_show_test_info();
ret = fi_getinfo(FT_FIVERSION, ft_strptr(test_info.node),
ft_strptr(test_info.service), FI_SOURCE,
hints, &info);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
} else {
if (info->next) {
printf("fi_getinfo returned multiple matches\n");
ret = -FI_E2BIG;
} else {
/* fabric_info is replaced when connecting */
fabric_info = info;
ret = ft_run_test();
if (fabric_info != info)
fi_freeinfo(fabric_info);
}
fi_freeinfo(info);
}
if (ret) {
printf("Node: %s\nService: %s\n",
test_info.node, test_info.service);
printf("%s\n", fi_tostr(hints, FI_TYPE_INFO));
}
printf("Ending test %d-%d, result: %s\n", test_info.test_index,
test_info.test_subindex, fi_strerror(-ret));
results[ft_fw_result_index(-ret)]++;
ret = ft_fw_send(sock, &ret, sizeof ret);
} while (!ret);
fi_freeinfo(hints);
return ret;
}
/*
* Tests:
* - synchronous resolution of bad address
*/
static int
av_bad_sync()
{
int testret;
int ret;
struct fid_av *av;
struct fi_av_attr attr;
uint8_t addrbuf[4096];
int buflen;
fi_addr_t fi_addr;
testret = FAIL;
memset(&attr, 0, sizeof(attr));
attr.type = av_type;
attr.count = 32;
av = NULL;
ret = fi_av_open(domain, &attr, &av, NULL);
if (ret != 0) {
sprintf(err_buf, "fi_av_open(%s) = %d, %s",
fi_tostr(&av_type, FI_TYPE_AV_TYPE),
ret, fi_strerror(-ret));
goto fail;
}
fi_addr = ~FI_ADDR_NOTAVAIL;
buflen = sizeof(addrbuf);
ret = av_create_address_list(bad_address, 0, 1, addrbuf, 0, buflen);
if (ret < 0) {
goto fail; // av_create_address_list filled err_buf
}
ret = fi_av_insert(av, addrbuf, 1, &fi_addr, 0, NULL);
if (ret != 0) {
sprintf(err_buf, "fi_av_insert ret=%d, should be 0", ret);
goto fail;
}
if (fi_addr != FI_ADDR_NOTAVAIL) {
sprintf(err_buf,
"fi_addr = 0x%lx, should be 0x%lx (FI_ADDR_NOTAVAIL)",
fi_addr, FI_ADDR_NOTAVAIL);
goto fail;
}
testret = PASS;
fail:
if (av != NULL) {
fi_close(&av->fid);
}
return testret;
}
//debug routine to print the opal_value_t returned by query interface
void print_transports_query()
{
opal_value_t *providers=NULL;
char* prov_name = NULL;
int ret;
int32_t *protocol_ptr, protocol;
int8_t conduit_id;
int8_t *prov_num=&conduit_id;
protocol_ptr = &protocol;
opal_output(0,"\n print_transports_query() Begin- %s:%d",__FILE__,__LINE__);
opal_output(0,"\n calling the orte_rml_ofi_query_transports() ");
if( ORTE_SUCCESS == orte_rml.query_transports(&providers)) {
opal_output(0,"\n query_transports() completed, printing details\n");
while (providers) {
//get the first opal_list_t;
opal_list_t temp;
opal_list_t *prov = &temp;
ret = opal_value_unload(providers,(void **)&prov,OPAL_PTR);
if (ret == OPAL_SUCCESS) {
opal_output_verbose(1,orte_rml_base_framework.framework_output,"\n %s:%d opal_value_unload() succeeded, opal_list* prov = %x",
__FILE__,__LINE__,prov);
if (orte_get_attribute( prov, ORTE_CONDUIT_ID, (void **)&prov_num,OPAL_UINT8)) {
opal_output(0," Provider conduit_id : %d",*prov_num);
}
if( orte_get_attribute( prov, ORTE_PROTOCOL, (void **)&protocol_ptr,OPAL_UINT32)) {
opal_output(0," Protocol : %s",fi_tostr(protocol_ptr,FI_TYPE_PROTOCOL));
}
if( orte_get_attribute( prov, ORTE_PROV_NAME, (void **)&prov_name ,OPAL_STRING)) {
opal_output(0," Provider name : %s",prov_name);
} else {
opal_output(0," Error in getting Provider name");
}
} else {
opal_output(0," %s:%d opal_value_unload() failed, opal_list* prov = %x",__FILE__,__LINE__,prov);
}
providers = (opal_value_t *)providers->super.opal_list_next;
// opal_output_verbose(1,orte_rml_base_framework.framework_output,"\n %s:%d -
// Moving on to next provider provders=%x",__FILE__,__LINE__,providers);
}
} else {
opal_output(0,"\n query_transports() returned Error ");
}
opal_output(0,"\n End of print_transports_query() from ofi_query_test.c \n");
//need to free all the providers here
}
/*
* Tests:
* - async good vector
*/
static int
av_zero_async()
{
int testret;
int ret;
struct fid_av *av;
struct fi_av_attr attr;
uint8_t addrbuf[4096];
uint32_t ctx;
fi_addr_t fi_addr[MAX_ADDR];
testret = FAIL;
memset(&attr, 0, sizeof(attr));
attr.type = av_type;
attr.count = 32;
attr.flags = FI_EVENT;
av = NULL;
ret = fi_av_open(domain, &attr, &av, NULL);
if (ret != 0) {
sprintf(err_buf, "fi_av_open(%s) = %d, %s",
fi_tostr(&av_type, FI_TYPE_AV_TYPE),
ret, fi_strerror(-ret));
goto fail;
}
ret = fi_bind(&av->fid, &eq->fid, 0);
if (ret != 0) {
sprintf(err_buf, "fi_bind() = %d, %s", ret, fi_strerror(-ret));
goto fail;
}
ret = fi_av_insert(av, addrbuf, 0, fi_addr, 0, &ctx);
if (ret != 0) {
sprintf(err_buf, "fi_av_insert ret=%d, should be 0", ret);
goto fail;
}
if (check_eq_sread(eq, &av->fid, &ctx, 0, 20000, 0) != 0) {
goto fail;
}
testret = PASS;
fail:
if (av != NULL) {
fi_close(&av->fid);
}
return testret;
}
static int run(struct fi_info *hints, char *node, char *port)
{
struct fi_info *cur;
struct fi_info *info;
int ret;
ret = fi_getinfo(FI_VERSION(1, 0), node, port, 0, hints, &info);
if (ret) {
printf("fi_getinfo %s\n", strerror(-ret));
return ret;
}
for (cur = info; cur; cur = cur->next)
printf("%s\n", fi_tostr(cur, FI_TYPE_INFO));
fi_freeinfo(info);
return EXIT_SUCCESS;
}
/*
* Create an address list
*/
static int
av_create_address_list(char *first_address, int base, int num_addr,
void *addr_array, int offset, int len)
{
int (*add_address)(char *, int, void *);
uint8_t *cur_addr;
int addrlen;
int ret;
int i;
switch (fi->addr_format) {
case FI_SOCKADDR:
case FI_SOCKADDR_IN:
addrlen = sizeof(struct sockaddr_in);
add_address = av_create_addr_sockaddr_in;
break;
default:
sprintf(err_buf, "test does not yet support %s",
fi_tostr(&fi->addr_format, FI_TYPE_ADDR_FORMAT));
return -FI_ENOSYS;
}
if (len < addrlen * (offset + num_addr)) {
sprintf(err_buf, "internal error, not enough room for %d addresses",
num_addr);
return -FI_ENOMEM;
}
cur_addr = addr_array;
cur_addr += offset * addrlen;
for (i = 0; i < num_addr; ++i) {
ret = add_address(first_address, base + i, cur_addr);
if (ret != 0) {
return ret;
}
cur_addr += addrlen;
}
return cur_addr - (uint8_t *)addr_array;
}
static int ft_fw_client(void)
{
struct fi_info *hints, *info;
int ret;
hints = fi_allocinfo();
if (!hints)
return -FI_ENOMEM;
for (fts_start(series, test_start_index);
!fts_end(series, test_end_index);
fts_next(series)) {
fts_cur_info(series, &test_info);
ft_fw_convert_info(hints, &test_info);
printf("Starting test %d / %d\n", test_info.test_index, series->test_count);
ret = fi_getinfo(FT_FIVERSION, ft_strptr(test_info.node),
ft_strptr(test_info.service), 0, hints, &info);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
} else {
ret = ft_fw_process_list(hints, info);
fi_freeinfo(info);
}
if (ret) {
fprintf(stderr, "Node: %s\nService: %s\n",
test_info.node, test_info.service);
fprintf(stderr, "%s\n", fi_tostr(hints, FI_TYPE_INFO));
}
printf("Ending test %d / %d, result: %s\n",
test_info.test_index, series->test_count, fi_strerror(-ret));
results[ft_fw_result_index(-ret)]++;
}
fi_freeinfo(hints);
return 0;
}
int main(int argc, char **argv)
{
int op, ret, option_index;
int use_hints = 0;
hints = fi_allocinfo();
if (!hints)
return EXIT_FAILURE;
hints->mode = ~0;
hints->domain_attr->mode = ~0;
hints->domain_attr->mr_mode = ~(FI_MR_BASIC | FI_MR_SCALABLE);
while ((op = getopt_long(argc, argv, "n:P:c:m:t:a:p:d:f:elhv", longopts,
&option_index)) != -1) {
switch (op) {
case 0:
/* there is no short variant only for --version */
if (ver) {
printf("%s: %s\n", argv[0], PACKAGE_VERSION);
printf("libfabric: %s\n", fi_tostr("1", FI_TYPE_VERSION));
printf("libfabric api: %d.%d\n",
FI_MAJOR_VERSION, FI_MINOR_VERSION);
return EXIT_SUCCESS;
}
goto print_help;
case 'n':
node = optarg;
break;
case 'P':
port = optarg;
break;
case 'c':
ret = tokparse(optarg, str2cap, &hints->caps);
if (ret)
goto out;
use_hints = 1;
break;
case 'm':
hints->mode = 0;
ret = tokparse(optarg, str2mode, &hints->mode);
if (ret)
goto out;
use_hints = 1;
break;
case 't':
ret = str2ep_type(optarg, &hints->ep_attr->type);
if (ret)
goto out;
use_hints = 1;
break;
case 'a':
ret = str2addr_format(optarg, &hints->addr_format);
if (ret)
goto out;
use_hints = 1;
break;
case 'p':
free(hints->fabric_attr->prov_name);
hints->fabric_attr->prov_name = strdup(optarg);
use_hints = 1;
break;
case 'd':
hints->domain_attr->name = strdup(optarg);
use_hints = 1;
break;
case 'f':
hints->fabric_attr->name = strdup(optarg);
use_hints = 1;
break;
case 'e':
env = 1;
break;
case 'l':
list_providers = 1;
break;
case 'v':
verbose = 1;
break;
case 'h':
default:
print_help:
printf("Usage: %s\n", argv[0]);
usage();
return EXIT_FAILURE;
}
}
ret = run(use_hints ? hints : NULL, node, port);
out:
fi_freeinfo(hints);
return -ret;
}
/*
* Tests:
* - async 2 vector, 1 good, 1 mix bad+good
*/
static int
av_goodbad_2vector_async()
{
int testret;
int ret;
int i;
struct fid_av *av;
struct fi_av_attr attr;
uint8_t addrbuf[4096];
uint32_t event;
uint32_t ctx[2];
uint8_t good[2];
uint8_t err;
struct fi_eq_entry entry;
int buflen;
fi_addr_t fi_addr[MAX_ADDR];
testret = FAIL;
memset(&attr, 0, sizeof(attr));
attr.type = av_type;
attr.count = 32;
attr.flags = FI_EVENT;
av = NULL;
ret = fi_av_open(domain, &attr, &av, NULL);
if (ret != 0) {
sprintf(err_buf, "fi_av_open(%s) = %d, %s",
fi_tostr(&av_type, FI_TYPE_AV_TYPE),
ret, fi_strerror(-ret));
goto fail;
}
ret = fi_bind(&av->fid, &eq->fid, 0);
if (ret != 0) {
sprintf(err_buf, "fi_bind() = %d, %s", ret, fi_strerror(-ret));
goto fail;
}
for (i = 0; i < MAX_ADDR; ++i) {
fi_addr[i] = FI_ADDR_NOTAVAIL;
}
fi_addr[1] = ~FI_ADDR_NOTAVAIL;
buflen = sizeof(addrbuf);
/* 1st vector is one good address */
ret = av_create_address_list(good_address, 0, 1, addrbuf, 0, buflen);
if (ret < 0) {
goto fail; // av_create_address_list filled err_buf
}
ret = fi_av_insert(av, addrbuf, 1, fi_addr, FI_MORE, &ctx[0]);
if (ret != 1) {
sprintf(err_buf, "fi_av_insert ret=%d, %s", ret, fi_strerror(-ret));
goto fail;
}
ctx[0] = 1;
/* second vector is one bad address followed by N-1 good ones */
ret = av_create_address_list(bad_address, 0, 1, addrbuf, 0, buflen);
if (ret < 0) {
goto fail; // av_create_address_list filled err_buf
}
if (num_good_addr > 1) {
ret = av_create_address_list(good_address, 1, num_good_addr - 1,
addrbuf, 1, buflen);
if (ret < 0) {
goto fail; // av_create_address_list filled err_buf
}
}
ret = fi_av_insert(av, addrbuf, num_good_addr, &fi_addr[1], 0, &ctx[1]);
if (ret != num_good_addr) {
sprintf(err_buf, "fi_av_insert ret=%d, %s", ret, fi_strerror(-ret));
goto fail;
}
ctx[1] = num_good_addr - 1;
/*
* A little tricky here because the good completions may come in any order,
* all we can far for sure is that error must come before good completion 2.
*/
memset(good, 0, sizeof(good));
err = 0;
for (i = 0; i < 3; ++i) {
ret = fi_eq_sread(eq, &event, &entry, sizeof(entry), 20000, 0);
if (ret == -FI_EAVAIL) {
if (good[1] > 0 || err > 0) {
sprintf(err_buf, "Unexpected error completion");
goto fail;
}
ret = check_eq_readerr(eq, &av->fid, &ctx[1], 0);
if (ret != 0) {
goto fail;
}
err = 1;
} else {
ret = check_eq_result(ret, event, &entry, &av->fid, NULL, ~0);
if (ret != 0) {
goto fail;
}
if (entry.context != &ctx[0] &&
entry.context != &ctx[1]) {
sprintf(err_buf, "bad context: %p", entry.context);
goto fail;
}
if (*(uint32_t *)(entry.context) == ~0) {
sprintf(err_buf, "duplicate context: %p", entry.context);
goto fail;
}
if (entry.context == &ctx[1] && err == 0) {
sprintf(err_buf, "2nd good comp before error");
goto fail;
}
if (*(uint32_t *)(entry.context) != entry.data) {
sprintf(err_buf, "count = %lu, should be %d", entry.data,
*(uint32_t *)(entry.context));
goto fail;
}
*(uint32_t *)(entry.context) = ~0;
}
}
ret = fi_eq_sread(eq, &event, &entry, sizeof(entry), 1000, 0);
if (ret != -FI_ETIMEDOUT) {
sprintf(err_buf, "too many events");
goto fail;
}
for (i = 0; i < num_good_addr + 1; ++i) {
if (i == 1) {
if (fi_addr[1] != FI_ADDR_NOTAVAIL) {
sprintf(err_buf, "fi_addr[1] != FI_ADDR_NOTAVAIL");
goto fail;
}
} else {
if (fi_addr[i] == FI_ADDR_NOTAVAIL) {
sprintf(err_buf, "fi_addr[%d] = FI_ADDR_NOTAVAIL", i);
goto fail;
}
}
}
testret = PASS;
fail:
if (av != NULL) {
fi_close(&av->fid);
}
return testret;
}
/*
* Tests:
* - async vector with 1 good and 1 bad
*/
static int
av_goodbad_vector_async()
{
int testret;
int ret;
int i;
struct fid_av *av;
struct fi_av_attr attr;
uint8_t addrbuf[4096];
uint32_t event;
uint32_t ctx;
struct fi_eq_entry entry;
int buflen;
fi_addr_t fi_addr[MAX_ADDR];
testret = FAIL;
memset(&attr, 0, sizeof(attr));
attr.type = av_type;
attr.count = 32;
attr.flags = FI_EVENT;
av = NULL;
ret = fi_av_open(domain, &attr, &av, NULL);
if (ret != 0) {
sprintf(err_buf, "fi_av_open(%s) = %d, %s",
fi_tostr(&av_type, FI_TYPE_AV_TYPE),
ret, fi_strerror(-ret));
goto fail;
}
ret = fi_bind(&av->fid, &eq->fid, 0);
if (ret != 0) {
sprintf(err_buf, "fi_bind() = %d, %s", ret, fi_strerror(-ret));
goto fail;
}
for (i = 0; i < MAX_ADDR; ++i) {
fi_addr[i] = FI_ADDR_NOTAVAIL;
}
fi_addr[1] = ~FI_ADDR_NOTAVAIL;
buflen = sizeof(addrbuf);
/* vector is good address + bad address */
ret = av_create_address_list(good_address, 0, 1, addrbuf, 0, buflen);
if (ret < 0) {
goto fail; // av_create_address_list filled err_buf
}
ret = av_create_address_list(bad_address, 0, 1, addrbuf, 1, buflen);
if (ret < 0) {
goto fail; // av_create_address_list filled err_buf
}
ret = fi_av_insert(av, addrbuf, 2, fi_addr, 0, &ctx);
if (ret != 2) {
sprintf(err_buf, "fi_av_insert ret=%d, %s", ret, fi_strerror(-ret));
goto fail;
}
/*
* Read event after sync, verify we get FI_EAVAIL, then read and
* verify the error completion
*/
ret = fi_eq_sread(eq, &event, &entry, sizeof(entry), 20000, 0);
if (ret != -FI_EAVAIL) {
sprintf(err_buf, "fi_eq_sread ret = %d, should be -FI_EAVAIL", ret);
goto fail;
}
ret = check_eq_readerr(eq, &av->fid, &ctx, 1);
if (ret != 0) {
goto fail;
}
/*
* Now we should get a good completion, and all fi_addr except fd_addr[1]
* should have good values.
*/
if (check_eq_sread(eq, &av->fid, &ctx, 1, 20000, 0) != 0) {
goto fail;
}
if (fi_addr[0] == FI_ADDR_NOTAVAIL) {
sprintf(err_buf, "fi_addr[0] = FI_ADDR_NOTAVAIL");
goto fail;
}
if (fi_addr[1] != FI_ADDR_NOTAVAIL) {
sprintf(err_buf, "fi_addr[1] != FI_ADDR_NOTAVAIL");
goto fail;
}
testret = PASS;
fail:
if (av != NULL) {
fi_close(&av->fid);
}
return testret;
}
/*
* Tests:
* - async 2 good vectors
*/
static int
av_good_2vector_async()
{
int testret;
int ret;
int i;
struct fid_av *av;
struct fi_av_attr attr;
uint8_t addrbuf[4096];
uint32_t event;
struct fi_eq_entry entry;
uint32_t ctx[2];
int buflen;
fi_addr_t fi_addr[MAX_ADDR];
testret = FAIL;
memset(&attr, 0, sizeof(attr));
attr.type = av_type;
attr.count = 32;
attr.flags = FI_EVENT;
av = NULL;
ret = fi_av_open(domain, &attr, &av, NULL);
if (ret != 0) {
sprintf(err_buf, "fi_av_open(%s) = %d, %s",
fi_tostr(&av_type, FI_TYPE_AV_TYPE),
ret, fi_strerror(-ret));
goto fail;
}
ret = fi_bind(&av->fid, &eq->fid, 0);
if (ret != 0) {
sprintf(err_buf, "fi_bind() = %d, %s", ret, fi_strerror(-ret));
goto fail;
}
for (i = 0; i < MAX_ADDR; ++i) {
fi_addr[i] = FI_ADDR_NOTAVAIL;
}
buflen = sizeof(addrbuf);
/* 1st vector is just first address */
ret = av_create_address_list(good_address, 0, 1, addrbuf, 0, buflen);
if (ret < 0) {
goto fail; // av_create_address_list filled err_buf
}
ret = fi_av_insert(av, addrbuf, 1, fi_addr, FI_MORE, &ctx[0]);
if (ret != 1) {
sprintf(err_buf, "fi_av_insert ret=%d, %s", ret, fi_strerror(-ret));
goto fail;
}
ctx[0] = 1;
/* 2nd vector is remaining addresses */
ret = av_create_address_list(good_address, 1, num_good_addr-1,
addrbuf, 0, buflen);
if (ret < 0) {
goto fail; // av_create_address_list filled err_buf
}
ret = fi_av_insert(av, addrbuf, num_good_addr-1, &fi_addr[1], 0, &ctx[1]);
if (ret != num_good_addr-1) {
sprintf(err_buf, "fi_av_insert ret=%d, %s", ret, fi_strerror(-ret));
goto fail;
}
ctx[1] = num_good_addr-1;
/*
* Handle completions in either order
*/
for (i = 0; i < 2; ++i) {
ret = fi_eq_sread(eq, &event, &entry, sizeof(entry), 20000, 0);
ret = check_eq_result(ret, event, &entry, &av->fid, NULL, ~0);
if (ret != 0) {
goto fail;
}
if (entry.context != &ctx[0] && entry.context != &ctx[1]) {
sprintf(err_buf, "bad context: %p", entry.context);
goto fail;
}
if (*(uint32_t *)(entry.context) == ~0) {
sprintf(err_buf, "duplicate context: %p", entry.context);
goto fail;
}
if (*(uint32_t *)(entry.context) != entry.data) {
sprintf(err_buf, "count = %lu, should be %d", entry.data,
*(uint32_t *)(entry.context));
goto fail;
}
*(uint32_t *)(entry.context) = ~0;
}
for (i = 0; i < num_good_addr; ++i) {
if (fi_addr[i] == FI_ADDR_NOTAVAIL) {
sprintf(err_buf, "fi_addr[%d] = FI_ADDR_NOTAVAIL", i);
goto fail;
}
}
testret = PASS;
fail:
if (av != NULL) {
fi_close(&av->fid);
}
return testret;
}
int main(int argc, char **argv)
{
int op, ret, option_index;
int use_hints = 0;
hints = fi_allocinfo();
if (!hints)
return EXIT_FAILURE;
hints->mode = ~0;
while ((op = getopt_long(argc, argv, "n:p:c:m:t:a:f:ehv", longopts, &option_index)) != -1) {
switch (op) {
case 0:
/* If --verbose set a flag, do nothing. */
if (longopts[option_index].flag != 0)
break;
case 'n':
node = optarg;
break;
case 'p':
port = optarg;
break;
case 'c':
hints->caps = tokparse(optarg, str2cap);
use_hints = 1;
break;
case 'm':
hints->mode = tokparse(optarg, str2mode);
use_hints = 1;
break;
case 't':
hints->ep_attr->type = str2ep_type(optarg);
use_hints = 1;
break;
case 'a':
hints->addr_format = str2addr_format(optarg);
use_hints = 1;
break;
case 'f':
hints->fabric_attr->prov_name = strdup(optarg);
use_hints = 1;
break;
case 'v':
verbose = 1;
break;
case 'h':
default:
printf("Usage: %s\n", argv[0]);
usage();
return EXIT_FAILURE;
}
}
if (ver) {
printf("%s: %s\n", argv[0], PACKAGE_VERSION);
printf("libfabric: %s\n", fi_tostr("1", FI_TYPE_VERSION));
printf("libfabric api: %d.%d\n", FI_MAJOR_VERSION, FI_MINOR_VERSION);
return EXIT_SUCCESS;
}
ret = run(use_hints ? hints : NULL, node, port);
fi_freeinfo(hints);
return -ret;
}
/*
* Tests:
* - async good vector
*/
static int
av_good_vector_async()
{
int testret;
int ret;
int i;
struct fid_av *av;
struct fi_av_attr attr;
uint8_t addrbuf[4096];
uint32_t ctx;
int buflen;
fi_addr_t fi_addr[MAX_ADDR];
testret = FAIL;
memset(&attr, 0, sizeof(attr));
attr.type = av_type;
attr.count = 32;
attr.flags = FI_EVENT;
av = NULL;
ret = fi_av_open(domain, &attr, &av, NULL);
if (ret != 0) {
sprintf(err_buf, "fi_av_open(%s) = %d, %s",
fi_tostr(&av_type, FI_TYPE_AV_TYPE),
ret, fi_strerror(-ret));
goto fail;
}
ret = fi_bind(&av->fid, &eq->fid, 0);
if (ret != 0) {
sprintf(err_buf, "fi_bind() = %d, %s", ret, fi_strerror(-ret));
goto fail;
}
for (i = 0; i < MAX_ADDR; ++i) {
fi_addr[i] = FI_ADDR_NOTAVAIL;
}
buflen = sizeof(addrbuf);
ret = av_create_address_list(good_address, 0, num_good_addr,
addrbuf, 0, buflen);
if (ret < 0) {
goto fail; // av_create_address_list filled err_buf
}
for (i = 0; i < num_good_addr; ++i) {
fi_addr[i] = FI_ADDR_NOTAVAIL;
}
ret = fi_av_insert(av, addrbuf, num_good_addr, fi_addr, 0, &ctx);
if (ret != num_good_addr) {
sprintf(err_buf, "fi_av_insert ret=%d, %s", ret, fi_strerror(-ret));
goto fail;
}
if (check_eq_sread(eq, &av->fid, &ctx, num_good_addr, 20000, 0) != 0) {
goto fail;
}
for (i = 0; i < num_good_addr; ++i) {
if (fi_addr[i] == FI_ADDR_NOTAVAIL) {
sprintf(err_buf, "fi_addr[%d] = FI_ADDR_NOTAVAIL", i);
goto fail;
}
}
testret = PASS;
fail:
if (av != NULL) {
fi_close(&av->fid);
}
return testret;
}
/*
* Tests:
* - sync vector with 1 good and 1 bad
*/
static int
av_goodbad_vector_sync()
{
int testret;
int ret;
int i;
struct fid_av *av;
struct fi_av_attr attr;
uint8_t addrbuf[4096];
int buflen;
fi_addr_t fi_addr[MAX_ADDR];
testret = FAIL;
memset(&attr, 0, sizeof(attr));
attr.type = av_type;
attr.count = 32;
av = NULL;
ret = fi_av_open(domain, &attr, &av, NULL);
if (ret != 0) {
sprintf(err_buf, "fi_av_open(%s) = %d, %s",
fi_tostr(&av_type, FI_TYPE_AV_TYPE),
ret, fi_strerror(-ret));
goto fail;
}
for (i = 0; i < MAX_ADDR; ++i) {
fi_addr[i] = FI_ADDR_NOTAVAIL;
}
fi_addr[1] = ~FI_ADDR_NOTAVAIL;
buflen = sizeof(addrbuf);
/* vector is good address + bad address */
ret = av_create_address_list(good_address, 0, 1, addrbuf, 0, buflen);
if (ret < 0) {
goto fail; // av_create_address_list filled err_buf
}
ret = av_create_address_list(bad_address, 0, 1, addrbuf, 1, buflen);
if (ret < 0) {
goto fail; // av_create_address_list filled err_buf
}
ret = fi_av_insert(av, addrbuf, 2, fi_addr, 0, NULL);
if (ret != 1) {
sprintf(err_buf, "fi_av_insert ret=%d, should be 1", ret);
goto fail;
}
/*
* Check returned fi_addrs
*/
if (fi_addr[0] == FI_ADDR_NOTAVAIL) {
sprintf(err_buf, "fi_addr[0] = FI_ADDR_NOTAVAIL");
goto fail;
}
if (fi_addr[1] != FI_ADDR_NOTAVAIL) {
sprintf(err_buf, "fi_addr[1] != FI_ADDR_NOTAVAIL");
goto fail;
}
testret = PASS;
fail:
if (av != NULL) {
fi_close(&av->fid);
}
return testret;
}
int _gnix_amo_post_req(void *data)
{
struct gnix_fab_req *fab_req = (struct gnix_fab_req *)data;
struct gnix_fid_ep *ep = fab_req->gnix_ep;
struct gnix_nic *nic = ep->nic;
struct gnix_fid_mem_desc *loc_md;
struct gnix_tx_descriptor *txd;
gni_mem_handle_t mdh;
gni_return_t status;
int rc;
int inject_err = _gnix_req_inject_err(fab_req);
if (!gnix_ops_allowed(ep, fab_req->vc->peer_caps, fab_req->flags)) {
GNIX_DEBUG(FI_LOG_EP_DATA, "flags:0x%llx, %s\n", fab_req->flags,
fi_tostr(&fab_req->flags, FI_TYPE_OP_FLAGS));
GNIX_DEBUG(FI_LOG_EP_DATA, "caps:0x%llx, %s\n",
ep->caps, fi_tostr(&ep->caps, FI_TYPE_CAPS));
GNIX_DEBUG(FI_LOG_EP_DATA, "peer_caps:0x%llx, %s\n",
fab_req->vc->peer_caps,
fi_tostr(&fab_req->vc->peer_caps, FI_TYPE_OP_FLAGS));
rc = __gnix_amo_post_err(fab_req, FI_EOPNOTSUPP);
if (rc != FI_SUCCESS)
GNIX_WARN(FI_LOG_EP_DATA,
"__gnix_amo_post_err() failed: %d\n", rc);
return -FI_ECANCELED;
}
rc = _gnix_nic_tx_alloc(nic, &txd);
if (rc) {
GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n",
rc);
return -FI_ENOSPC;
}
txd->completer_fn = __gnix_amo_txd_complete;
txd->req = fab_req;
/* Mem handle CRC is not validated during FMA operations. Skip this
* costly calculation. */
_gnix_convert_key_to_mhdl_no_crc(
(gnix_mr_key_t *)&fab_req->amo.rem_mr_key,
&mdh);
loc_md = (struct gnix_fid_mem_desc *)fab_req->amo.loc_md;
txd->gni_desc.type = GNI_POST_AMO;
txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */
txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */
txd->gni_desc.local_addr = (uint64_t)fab_req->amo.loc_addr;
if (loc_md) {
txd->gni_desc.local_mem_hndl = loc_md->mem_hndl;
}
txd->gni_desc.remote_addr = (uint64_t)fab_req->amo.rem_addr;
txd->gni_desc.remote_mem_hndl = mdh;
txd->gni_desc.length = fab_req->amo.len;
txd->gni_desc.rdma_mode = 0; /* check flags */
txd->gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */
txd->gni_desc.amo_cmd = _gnix_atomic_cmd(fab_req->amo.datatype,
fab_req->amo.op,
fab_req->type);
txd->gni_desc.first_operand = fab_req->amo.first_operand;
txd->gni_desc.second_operand = fab_req->amo.second_operand;
GNIX_DEBUG(FI_LOG_EP_DATA, "fo:%016lx so:%016lx\n",
txd->gni_desc.first_operand, txd->gni_desc.second_operand);
GNIX_DEBUG(FI_LOG_EP_DATA, "amo_cmd:%x\n",
txd->gni_desc.amo_cmd);
GNIX_LOG_DUMP_TXD(txd);
COND_ACQUIRE(nic->requires_lock, &nic->lock);
if (OFI_UNLIKELY(inject_err)) {
_gnix_nic_txd_err_inject(nic, txd);
status = GNI_RC_SUCCESS;
} else {
status = GNI_PostFma(fab_req->vc->gni_ep, &txd->gni_desc);
}
COND_RELEASE(nic->requires_lock, &nic->lock);
if (status != GNI_RC_SUCCESS) {
_gnix_nic_tx_free(nic, txd);
GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n",
gni_err_str[status]);
}
return gnixu_to_fi_errno(status);
}
static void libfabric_init() {
int i;
struct fi_info *info = NULL;
struct fi_info *hints = fi_allocinfo();
struct fi_av_attr av_attr = {0};
struct fi_cq_attr cq_attr = {0};
int max_tx_ctx, max_rx_ctx;
int comm_concurrency;
int rx_ctx_cnt;
int rx_ctx_bits = 0;
hints->mode = ~0;
hints->caps = FI_RMA
| FI_ATOMIC
| FI_SOURCE /* do we want this? */
| FI_READ
| FI_WRITE
| FI_REMOTE_READ
| FI_REMOTE_WRITE
| FI_MULTI_RECV
| FI_FENCE;
hints->addr_format = FI_FORMAT_UNSPEC;
#if defined(CHPL_COMM_SUBSTRATE_SOCKETS)
//
// fi_freeinfo(hints) will free() hints->fabric_attr->prov_name; this
// is documented, though poorly. So, get that space from malloc().
//
{
const char s[] = "sockets";
char* sDup = sys_malloc(sizeof(s));
strcpy(sDup, s);
hints->fabric_attr->prov_name = sDup;
}
#elif defined(CHPL_COMM_SUBSTRATE_GNI)
#error "Substrate GNI not supported"
#else
#error "Substrate type not supported"
#endif
/* connectionless reliable */
hints->ep_attr->type = FI_EP_RDM;
hints->domain_attr->threading = FI_THREAD_UNSPEC;
hints->domain_attr->control_progress = FI_PROGRESS_MANUAL;
hints->domain_attr->data_progress = FI_PROGRESS_MANUAL;
hints->domain_attr->av_type = FI_AV_TABLE;
hints->domain_attr->mr_mode = FI_MR_SCALABLE;
hints->domain_attr->resource_mgmt = FI_RM_ENABLED;
// hints->domain_attr->cq_data_size
hints->tx_attr->op_flags = FI_COMPLETION;
hints->rx_attr->op_flags = FI_COMPLETION;
OFICHKERR(fi_getinfo(FI_VERSION(1,0), NULL, NULL, 0, hints, &info));
if (info == NULL) {
chpl_internal_error("No fabrics detected.");
} else {
#ifdef PRINT_FI_GETINFO
struct fi_info *cur;
for (cur = info; cur; cur = cur->next) {
printf("---\n");
printf("%s", fi_tostr(cur, FI_TYPE_INFO));
}
printf("\n");
#endif
}
ofi.num_am_ctx = 1; // Would we ever want more?
max_tx_ctx = info->domain_attr->max_ep_tx_ctx;
max_rx_ctx = info->domain_attr->max_ep_rx_ctx;
comm_concurrency = get_comm_concurrency();
ofi.num_tx_ctx = comm_concurrency+ofi.num_am_ctx > max_tx_ctx ?
max_tx_ctx-ofi.num_am_ctx : comm_concurrency;
ofi.num_rx_ctx = comm_concurrency+ofi.num_am_ctx > max_rx_ctx ?
max_rx_ctx-ofi.num_am_ctx : comm_concurrency;
info->ep_attr->tx_ctx_cnt = ofi.num_tx_ctx + ofi.num_am_ctx;
info->ep_attr->rx_ctx_cnt = ofi.num_rx_ctx + ofi.num_am_ctx;
OFICHKERR(fi_fabric(info->fabric_attr, &ofi.fabric, NULL));
OFICHKERR(fi_domain(ofi.fabric, info, &ofi.domain, NULL));
rx_ctx_cnt = ofi.num_rx_ctx + ofi.num_am_ctx;
while (rx_ctx_cnt >> ++rx_ctx_bits);
av_attr.rx_ctx_bits = rx_ctx_bits;
av_attr.type = FI_AV_TABLE;
av_attr.count = chpl_numNodes;
OFICHKERR(fi_av_open(ofi.domain, &av_attr, &ofi.av, NULL));
OFICHKERR(fi_scalable_ep(ofi.domain, info, &ofi.ep, NULL));
OFICHKERR(fi_scalable_ep_bind(ofi.ep, &ofi.av->fid, 0));
/* set up tx and rx contexts */
cq_attr.format = FI_CQ_FORMAT_CONTEXT;
cq_attr.size = 1024; /* ??? */
cq_attr.wait_obj = FI_WAIT_UNSPEC;
ofi.tx_ep = (struct fid_ep **) chpl_mem_allocMany(ofi.num_tx_ctx,
sizeof(ofi.tx_ep[0]),
CHPL_RT_MD_COMM_PER_LOC_INFO,
0, 0);
ofi.tx_cq = (struct fid_cq **) chpl_mem_allocMany(ofi.num_tx_ctx,
sizeof(ofi.tx_cq[0]),
CHPL_RT_MD_COMM_PER_LOC_INFO,
0, 0);
for (i = 0; i < ofi.num_tx_ctx; i++) {
OFICHKERR(fi_tx_context(ofi.ep, i, NULL, &ofi.tx_ep[i], NULL));
OFICHKERR(fi_cq_open(ofi.domain, &cq_attr, &ofi.tx_cq[i], NULL));
OFICHKERR(fi_ep_bind(ofi.tx_ep[i], &ofi.tx_cq[i]->fid, FI_TRANSMIT));
OFICHKERR(fi_enable(ofi.tx_ep[i]));
}
ofi.rx_ep = (struct fid_ep **) chpl_mem_allocMany(ofi.num_rx_ctx,
sizeof(ofi.rx_ep[0]),
CHPL_RT_MD_COMM_PER_LOC_INFO,
0, 0);
ofi.rx_cq = (struct fid_cq **) chpl_mem_allocMany(ofi.num_rx_ctx,
sizeof(ofi.rx_cq[0]),
CHPL_RT_MD_COMM_PER_LOC_INFO,
0, 0);
for (i = 0; i < ofi.num_rx_ctx; i++) {
OFICHKERR(fi_rx_context(ofi.ep, i, NULL, &ofi.rx_ep[i], NULL));
OFICHKERR(fi_cq_open(ofi.domain, &cq_attr, &ofi.rx_cq[i], NULL));
OFICHKERR(fi_ep_bind(ofi.rx_ep[i], &ofi.rx_cq[i]->fid, FI_RECV));
OFICHKERR(fi_enable(ofi.rx_ep[i]));
}
ofi.am_tx_ep = (struct fid_ep **) chpl_mem_allocMany(ofi.num_am_ctx,
sizeof(ofi.am_tx_ep[0]),
CHPL_RT_MD_COMM_PER_LOC_INFO,
0, 0);
ofi.am_tx_cq = (struct fid_cq **) chpl_mem_allocMany(ofi.num_am_ctx,
sizeof(ofi.am_tx_cq[0]),
CHPL_RT_MD_COMM_PER_LOC_INFO,
0, 0);
/* set up AM contexts */
for (i = 0; i < ofi.num_am_ctx; i++) {
OFICHKERR(fi_tx_context(ofi.ep, i+ofi.num_tx_ctx, NULL, &ofi.am_tx_ep[i], NULL));
OFICHKERR(fi_cq_open(ofi.domain, &cq_attr, &ofi.am_tx_cq[i], NULL));
OFICHKERR(fi_ep_bind(ofi.am_tx_ep[i], &ofi.am_tx_cq[i]->fid, FI_TRANSMIT));
OFICHKERR(fi_enable(ofi.am_tx_ep[i]));
}
ofi.am_rx_ep = (struct fid_ep **) chpl_mem_allocMany(ofi.num_am_ctx,
sizeof(ofi.am_rx_ep[0]),
CHPL_RT_MD_COMM_PER_LOC_INFO,
0, 0);
ofi.am_rx_cq = (struct fid_cq **) chpl_mem_allocMany(ofi.num_am_ctx,
sizeof(ofi.am_rx_cq[0]),
CHPL_RT_MD_COMM_PER_LOC_INFO,
0, 0);
for (i = 0; i < ofi.num_am_ctx; i++) {
OFICHKERR(fi_rx_context(ofi.ep, i+ofi.num_rx_ctx, NULL, &ofi.am_rx_ep[i], NULL));
OFICHKERR(fi_cq_open(ofi.domain, &cq_attr, &ofi.am_rx_cq[i], NULL));
OFICHKERR(fi_ep_bind(ofi.am_rx_ep[i], &ofi.am_rx_cq[i]->fid, FI_RECV));
OFICHKERR(fi_enable(ofi.am_rx_ep[i]));
}
OFICHKERR(fi_enable(ofi.ep));
libfabric_init_addrvec(rx_ctx_cnt, rx_ctx_bits);
OFICHKERR(fi_mr_reg(ofi.domain, 0, SIZE_MAX,
FI_READ | FI_WRITE | FI_REMOTE_READ | FI_REMOTE_WRITE |
FI_SEND | FI_RECV, 0,
(uint64_t) chpl_nodeID, 0, &ofi.mr, NULL));
fi_freeinfo(info); /* No error returned */
fi_freeinfo(hints); /* No error returned */
chpl_msg(2, "%d: completed libfabric initialization\n", chpl_nodeID);
}
static int _gnix_ep_getinfo(enum fi_ep_type ep_type, uint32_t version,
const char *node, const char *service,
uint64_t flags, struct fi_info *hints,
struct fi_info **info)
{
uint64_t mode = GNIX_FAB_MODES;
struct fi_info *gnix_info = NULL;
int ret = -FI_ENODATA;
GNIX_TRACE(FI_LOG_FABRIC, "\n");
if ((hints && hints->ep_attr) &&
(hints->ep_attr->type != FI_EP_UNSPEC &&
hints->ep_attr->type != ep_type)) {
return -FI_ENODATA;
}
gnix_info = _gnix_allocinfo();
if (!gnix_info)
return -FI_ENOMEM;
gnix_info->ep_attr->type = ep_type;
if (hints) {
/* TODO: Add version check when we decide on how to do it */
if (hints->addr_format == FI_ADDR_STR) {
gnix_info->addr_format = FI_ADDR_STR;
}
if (hints->ep_attr) {
/* Only support FI_PROTO_GNI protocol. */
switch (hints->ep_attr->protocol) {
case FI_PROTO_UNSPEC:
case FI_PROTO_GNI:
break;
default:
goto err;
}
if ((hints->ep_attr->tx_ctx_cnt > GNIX_SEP_MAX_CNT) &&
(hints->ep_attr->tx_ctx_cnt !=
FI_SHARED_CONTEXT)) {
goto err;
}
if (hints->ep_attr->rx_ctx_cnt > GNIX_SEP_MAX_CNT)
goto err;
if (hints->ep_attr->tx_ctx_cnt)
gnix_info->ep_attr->tx_ctx_cnt =
hints->ep_attr->tx_ctx_cnt;
if (hints->ep_attr->rx_ctx_cnt)
gnix_info->ep_attr->rx_ctx_cnt =
hints->ep_attr->rx_ctx_cnt;
if (hints->ep_attr->max_msg_size > GNIX_MAX_MSG_SIZE)
goto err;
}
GNIX_DEBUG(FI_LOG_FABRIC, "Passed EP attributes check\n");
/*
* check the mode field
*/
if (hints->mode) {
if ((hints->mode & GNIX_FAB_MODES) != GNIX_FAB_MODES) {
goto err;
}
mode = hints->mode & ~GNIX_FAB_MODES_CLEAR;
}
GNIX_DEBUG(FI_LOG_FABRIC, "Passed mode check\n");
if (hints->caps) {
/* The provider must support all requested
* capabilities. */
if ((hints->caps & GNIX_EP_CAPS_FULL) != hints->caps)
goto err;
/* The provider may silently enable secondary
* capabilities that do not introduce any overhead. */
gnix_info->caps = hints->caps | GNIX_EP_SEC_CAPS;
}
GNIX_DEBUG(FI_LOG_FABRIC, "Passed caps check gnix_info->caps = 0x%016lx\n",
gnix_info->caps);
if (hints->tx_attr) {
if ((hints->tx_attr->op_flags & GNIX_EP_OP_FLAGS) !=
hints->tx_attr->op_flags) {
goto err;
}
if (hints->tx_attr->inject_size > GNIX_INJECT_SIZE) {
goto err;
}
gnix_info->tx_attr->op_flags =
hints->tx_attr->op_flags & GNIX_EP_OP_FLAGS;
}
GNIX_DEBUG(FI_LOG_FABRIC, "Passed TX attributes check\n");
if (hints->rx_attr) {
if ((hints->rx_attr->op_flags & GNIX_EP_OP_FLAGS) !=
hints->rx_attr->op_flags) {
goto err;
}
gnix_info->rx_attr->op_flags =
hints->rx_attr->op_flags & GNIX_EP_OP_FLAGS;
}
if (hints->fabric_attr && hints->fabric_attr->name &&
strncmp(hints->fabric_attr->name, gnix_fab_name,
strlen(gnix_fab_name))) {
goto err;
}
GNIX_DEBUG(FI_LOG_FABRIC, "Passed fabric name check\n");
if (hints->domain_attr) {
if (hints->domain_attr->name &&
strncmp(hints->domain_attr->name, gnix_dom_name,
strlen(gnix_dom_name))) {
goto err;
}
if (hints->domain_attr->control_progress !=
FI_PROGRESS_UNSPEC)
gnix_info->domain_attr->control_progress =
hints->domain_attr->control_progress;
if (hints->domain_attr->data_progress !=
FI_PROGRESS_UNSPEC)
gnix_info->domain_attr->data_progress =
hints->domain_attr->data_progress;
switch (hints->domain_attr->mr_mode) {
case FI_MR_UNSPEC:
case FI_MR_BASIC:
if (FI_VERSION_GE(version, FI_VERSION(1, 5))) {
hints->domain_attr->mr_mode =
OFI_MR_BASIC_MAP;
}
break;
case FI_MR_SCALABLE:
GNIX_WARN(FI_LOG_FABRIC,
"GNI provider doesn't currently support MR_SCALABLE\n");
goto err;
}
switch (hints->domain_attr->threading) {
case FI_THREAD_COMPLETION:
gnix_info->domain_attr->threading =
hints->domain_attr->threading;
break;
default:
break;
}
if (hints->domain_attr->caps) {
if (hints->domain_attr->caps & ~GNIX_DOM_CAPS) {
GNIX_WARN(FI_LOG_FABRIC,
"Invalid domain caps\n");
goto err;
}
gnix_info->domain_attr->caps =
hints->domain_attr->caps;
}
ret = ofi_check_domain_attr(&gnix_prov, version,
gnix_info->domain_attr,
hints->domain_attr);
if (ret) {
GNIX_WARN(FI_LOG_FABRIC,
"GNI failed domain attributes check\n");
goto err;
}
GNIX_DEBUG(FI_LOG_FABRIC, "Passed the domain attributes check\n");
}
}
ret = __gnix_getinfo_resolve_node(node, service, flags, hints,
gnix_info);
if (ret != FI_SUCCESS)
goto err;
gnix_info->mode = mode;
gnix_info->fabric_attr->name = strdup(gnix_fab_name);
gnix_info->tx_attr->caps = gnix_info->caps;
gnix_info->tx_attr->mode = gnix_info->mode;
gnix_info->rx_attr->caps = gnix_info->caps;
gnix_info->rx_attr->mode = gnix_info->mode;
*info = gnix_info;
GNIX_DEBUG(FI_LOG_FABRIC, "Returning EP type: %s\n",
fi_tostr(&ep_type, FI_TYPE_EP_TYPE));
return FI_SUCCESS;
err:
fi_freeinfo(gnix_info);
return ret;
}
static void setup_ofi(const char *node, const char *service,
uint64_t flags)
{
struct fi_fabric_attr fabric_attr;
memset(&fabric_attr, 0, sizeof(fabric_attr));
fabric_attr.prov_name = (char*) "sockets";
//fabric_attr.prov_name = (char*) "usnic";
struct fi_ep_attr ep_attr;
memset(&ep_attr, 0, sizeof(ep_attr));
ep_attr.type = FI_EP_MSG;
struct fi_info hints;
memset(&hints, 0, sizeof(hints));
hints.caps = FI_MSG;
hints.mode = FI_LOCAL_MR;
hints.addr_format = FI_SOCKADDR_IN;
hints.ep_attr = &ep_attr;
hints.fabric_attr = &fabric_attr;
/* Get a minimum of libfabric v1.3.0. There were bugs in prior
versions; might as well start with the current libfabric
version. */
uint32_t libfabric_api;
libfabric_api = FI_VERSION(1, 3);
int ret;
ret = fi_getinfo(libfabric_api, node, service, flags,
&hints, &fidev.info);
if (0 != ret) {
error("cannot fi_getinfo");
}
int num_devs = 0;
for (struct fi_info *info = fidev.info;
NULL != info; info = info->next) {
++num_devs;
}
if (0 == num_devs) {
error("no fi devices available");
}
printf("INFO: %s\n", fi_tostr(fidev.info, FI_TYPE_INFO));
// Just use the first info returned
ret = fi_fabric(fidev.info->fabric_attr, &fidev.fabric, NULL);
if (0 != ret) {
error("fi_fabric failed");
}
// Make an EQ
struct fi_eq_attr eq_attr;
memset(&eq_attr, 0, sizeof(eq_attr));
#if WANT_FDS
eq_attr.wait_obj = FI_WAIT_FD;
#else
eq_attr.wait_obj = FI_WAIT_UNSPEC;
#endif
ret = fi_eq_open(fidev.fabric, &eq_attr, &fidev.eq, NULL);
if (0 != ret) {
error("fi_eq failed");
}
#if WANT_FDS
// Get the fd associated with this EQ
ret = fi_control(&(fidev.eq->fid), FI_GETWAIT, &fidev.eq_fd);
if (ret < 0) {
error("fi_control to get eq fq failed");
}
#endif
ret = fi_domain(fidev.fabric, fidev.info, &fidev.domain, NULL);
if (0 != ret) {
error("fi_domain failed");
}
#if WANT_FDS
// Make an epoll fd to listen on
epoll_fd = epoll_create(4096);
if (epoll_fd < 0) {
error("epoll_create failed");
}
#endif
}
