int openib_initialize()
{
// Use the previously cached info
me = l_state.rank;
nprocs = l_state.size;
assert(l_state.world_comm);
// initialize the envs
openib_init_envs();
//Initialize the registration cache
reg_cache_init(nprocs, 0);
init_params();
if(open_hca()) {
release_resources();
exit(1);
}
if(create_cq()) {
release_resources();
exit(1);
}
if(get_lid()) {
release_resources();
exit(1);
}
if(create_qp()) {
release_resources();
exit(1);
}
if(exch_addr()) {
release_resources();
exit(1);
}
if(connect_qp()) {
release_resources();
exit(1);
}
// Create network locks
openib_create_locks();
// Allocate buffers for one sided operations
openib_alloc_buf();
MPI_Barrier(l_state.world_comm);
return 0;
}
DictionaryDatum
Node::get_status_base()
{
DictionaryDatum dict = get_status_dict_();
assert( dict.valid() );
// add information available for all nodes
( *dict )[ names::local ] = is_local();
( *dict )[ names::model ] = LiteralDatum( get_name() );
// add information available only for local nodes
if ( is_local() )
{
( *dict )[ names::global_id ] = get_gid();
( *dict )[ names::frozen ] = is_frozen();
( *dict )[ names::node_uses_wfr ] = node_uses_wfr();
( *dict )[ names::thread ] = get_thread();
( *dict )[ names::vp ] = get_vp();
if ( parent_ )
{
( *dict )[ names::parent ] = parent_->get_gid();
// LIDs are only sensible for nodes with parents.
// Add 1 as we count lids internally from 0, but from
// 1 in the user interface.
( *dict )[ names::local_id ] = get_lid() + 1;
}
}
( *dict )[ names::thread_local_id ] = get_thread_lid();
( *dict )[ names::supports_precise_spikes ] = is_off_grid();
// This is overwritten with a corresponding value in the
// base classes for stimulating and recording devices, and
// in other special node classes
( *dict )[ names::element_type ] = LiteralDatum( names::neuron );
// now call the child class' hook
get_status( dict );
assert( dict.valid() );
return dict;
}
void __pc_ibv_post_send(struct ibv_qp *qp, struct ibv_send_wr *wr,
struct ibv_send_wr **bad_wr)
{
struct pc_hca *hca;
uint16_t lid;
double s, e;
s = get_dtime();
if (init_hcas_q == 0) {
return;
}
if (wr->opcode == IBV_WR_RDMA_WRITE)
{
lid = get_lid(qp->context);
hca = get_pc_hca(lid);
if (hca == NULL) {
return ;
}
struct pc_length *len = (struct pc_length*)malloc(sizeof(struct pc_length));
len->ctx = qp->context;
len->length = wr->sg_list->length;
lq_enq(&pc_q, len);
pthread_mutex_lock(&pc_mutex);
hca->co->pdg_num++;
hca->co->pdg_size += wr->sg_list->length;
pthread_mutex_unlock(&pc_mutex);
#ifdef DEBUG
fprintf(stderr, "[ %f ] post_send : pdg_num: %lu , pdg_size: %lu\n", get_dtime(), hca->co->pdg_num, hca->co->pdg_size);
#ifdef DETAIL
fprintf(stderr, "\tqp:%p, send_cq: %p, recv_cq: %p, context: %p, qp_context: %p\n", qp, qp->send_cq, qp->recv_cq, qp->context, qp->qp_context);
fprintf(stderr, "\twr_id:%lu, length:%d, opcode:%d, send_flags%d, imm_data:%u, remote_add:%lu, rkey:%u, xrc_remote_srq_num:%u\n", wr->wr_id, wr->sg_list->length, wr->opcode, wr->send_flags, wr->imm_data, wr->wr.rdma.remote_addr, wr->wr.rdma.rkey, wr->xrc_remote_srq_num);
fprintf(stderr, "\ttime:%f\n", get_dtime() - s);
#endif
#endif
}
return;
}
void __pc_ibv_poll_cq(struct ibv_cq *cq, int num_entries, struct ibv_wc *wc)
{
struct pc_length *len;
if (init_hcas_q == 0) {
return;
}
if (wc->opcode == IBV_WC_RDMA_WRITE) {
lq_init_it(&pc_q);
while ((len = (struct pc_length*)lq_next(&pc_q)) != NULL) {
struct pc_hca *hca;
uint16_t lid;
if (len->ctx == cq->context) {
lid = get_lid(cq->context);
hca = get_pc_hca(lid);
pthread_mutex_lock(&pc_mutex);
hca->co->pdg_num--;
hca->co->pdg_size = hca->co->pdg_size - len->length;
pthread_mutex_unlock(&pc_mutex);
lq_remove(&pc_q, len);
free(len);
lq_fin_it(&pc_q);
#ifdef DEBUG
fprintf(stderr, "[ %f ] post_cq : pdg_num: %lu , pdg_size: %lu \n", get_dtime(), hca->co->pdg_num, hca->co->pdg_size);
#ifdef DETAIL
struct ibv_port_attr pattr;
ibv_query_port(cq->context, 1, &pattr);
fprintf(stderr, "\tcp:%p, context:%p(more_ops:%p, abi_compat:%p, num_comp_vectors:%d, slid:%u), :\n", cq, cq->context, cq->context->more_ops, cq->context->abi_compat, cq->context->num_comp_vectors, pattr.lid);
fprintf(stderr, "\twr_id:%lu, status:%d, wc->opcode:%d, vendor_err:%u, byte_len:%u, imm_data:%u, qp_num:%p src_qp:%p, wc_flags:%d, pkey_index:%u, slid:%u, sl:%u, dlid_path_bits:%u, num_entries:%d \n", wc->wr_id, wc->status, wc->opcode, wc->vendor_err, wc->byte_len, wc->imm_data, wc->qp_num, wc->src_qp, wc->wc_flags, wc->pkey_index, wc->slid, wc->sl, wc->dlid_path_bits, num_entries);
#endif
#endif
return;
}
}
lq_fin_it(&pc_q);
}
return;
}
std::string
nest::Subnet::print_network( int max_depth, int level, std::string prefix )
{
// When the function is first called, we have to have a single
// space as prefix, otherwise everything will by slightly out of
// format.
if ( prefix == "" )
prefix = " ";
std::ostringstream out;
if ( get_parent() )
{
out << "+-[" << get_lid() + 1 << "] ";
if ( get_label() != "" )
out << get_label();
else
out << get_name();
}
else
{
out << "+-"
<< "[0] ";
if ( get_label() != "" )
out << get_label();
else
out << "root";
}
std::vector< int > dim;
get_dimensions_( dim );
out << " dim=[";
for ( size_t k = 0; k < dim.size() - 1; ++k )
out << dim[ k ] << " ";
out << dim[ dim.size() - 1 ] << "]" << std::endl;
if ( max_depth <= level )
return out.str();
if ( nodes_.empty() )
return out.str();
prefix += " ";
out << prefix << "|" << std::endl;
size_t first = 0;
for ( size_t i = 0; i < nodes_.size(); ++i )
{
size_t next = i + 1;
if ( nodes_[ i ] == NULL )
{
out << prefix << "+-NULL" << std::endl;
// Print extra line, if we are at the end of a subnet.
if ( next == nodes_.size() )
out << prefix << std::endl;
first = i + 1;
continue;
}
Subnet* c = dynamic_cast< Subnet* >( nodes_[ i ] );
if ( c != NULL )
{
// this node is a subnet,
// the sequence is printed, so
// we print the children and move on
// print subnet
//
// If the subnet is the last node of the parent subnet,
// we must not print the continuation line '|', so we distinguish
// this case.
if ( next == nodes_.size() )
out << prefix << nodes_[ i ]->print_network( max_depth, level + 1, prefix + " " );
else
out << prefix << nodes_[ i ]->print_network( max_depth, level + 1, prefix + "|" );
first = next;
continue;
}
// now we look one into the future
// to determine whether this is a sequence
// or not.
if ( next < nodes_.size() )
{
// we have a successor
if ( nodes_[ next ] != NULL )
{
// it is not NULL
c = dynamic_cast< Subnet* >( nodes_[ next ] );
if ( c == NULL )
{
// and not a subnet, so we skipp
// the printout, until the end
// of the sequence is found.
if ( ( nodes_[ first ]->get_name() == nodes_[ next ]->get_name() ) )
{
continue;
}
} // if the next node is a compount we flush the sequence
} // if the next node is NULL, we flush the sequence
} // if there is no next node, we flush the sequence
if ( first < i )
{
// Here we print the sequence of consecutive nodes.
// We can be sure that neither first, nor i point to NULL.
out << prefix << "+-[" << first + 1 << "]...[" << i + 1 << "] " << nodes_[ first ]->get_name()
<< std::endl;
// Print extra line, if we are at the end of a subnet.
if ( next == nodes_.size() )
out << prefix << std::endl;
first = next;
continue;
}
// Here, we deal the case of an individual Node with no identical neighbours.
out << prefix << "+-[" << i + 1 << "] " << nodes_[ first ]->get_name() << std::endl;
// Print extra line, if we are at the end of a subnet.
if ( next == nodes_.size() )
out << prefix << std::endl;
first = next;
}
return out.str();
}
