void mds_set_type_links(struct mds_net* net, struct mds* m,
int t, struct mds_links* ln)
{
unsigned i;
unsigned j;
unsigned* in;
struct mds_copy c;
PCU_Comm_Begin();
for (i = 0; i < ln->np; ++i) {
PCU_ALWAYS_ASSERT(ln->l);
for (j = 0; j < ln->n[i]; ++j)
PCU_COMM_PACK(ln->p[i], ln->l[i][j]);
}
PCU_Comm_Send();
while (PCU_Comm_Listen()) {
c.p = PCU_Comm_Sender();
PCU_ALWAYS_ASSERT(c.p != PCU_Comm_Self());
i = find_peer(ln, c.p);
in = PCU_Comm_Extract(ln->n[i] * sizeof(unsigned));
for (j = 0; j < ln->n[i]; ++j) {
c.e = mds_identify(t, in[j]);
mds_add_copy(net, m, mds_identify(t, ln->l[i][j]), c);
}
}
}
void mds_get_type_links(struct mds_net* net, struct mds* m,
int t, struct mds_links* ln)
{
unsigned i;
/* count remote links in np, p and n */
for_type_net(net, m, t, note_remote_link, ln);
alloc_links(ln);
for (i = 0; i < ln->np; ++i)
if (((unsigned)PCU_Comm_Self()) < ln->p[i])
/* zero n's for behavior in take_copy */
ln->n[i] = 0;
/* record indices in local order for owned boundaries */
for_type_net(net, m, t, take_remote_link, ln);
PCU_Comm_Begin();
/* send indices in local order for owned boundaries */
for_type_net(net, m, t, send_remote_link, NULL);
PCU_Comm_Send();
while (PCU_Comm_Listen())
/* recv indices in remote order for non-owned boundaries */
recv_links(ln);
}
/* algorithm courtesy of Sebastian Rettenberger:
use brokers/routers for the vertex global ids.
Although we have used this trick before (see mpas/apfMPAS.cc),
I didn't think to use it here, so credit is given. */
static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
{
Gid max = getMax(globalToVert);
Gid total = max + 1;
int peers = PCU_Comm_Peers();
int quotient = total / peers;
int remainder = total % peers;
int mySize = quotient;
int self = PCU_Comm_Self();
if (self == (peers - 1))
mySize += remainder;
typedef std::vector< std::vector<int> > TmpParts;
TmpParts tmpParts(mySize);
/* if we have a vertex, send its global id to the
broker for that global id */
PCU_Comm_Begin();
APF_ITERATE(GlobalToVert, globalToVert, it) {
int gid = it->first;
int to = std::min(peers - 1, gid / quotient);
PCU_COMM_PACK(to, gid);
}
static int align_copies(struct mds_net* net, struct mds* m)
{
int d;
mds_id e;
struct mds_copies* c;
int did_change = 0;
PCU_Comm_Begin();
for (d = 1; d < m->d; ++d)
for (e = mds_begin(m, d); e != MDS_NONE; e = mds_next(m, e)) {
c = mds_get_copies(net, e);
if (!c)
continue;
if (owns_copies(e, c))
downs_to_copies(m, e, c);
}
PCU_Comm_Send();
while (PCU_Comm_Receive())
if (recv_down_copies(net, m))
did_change = 1;
return PCU_Or(did_change);
}
/* see apf/apfConvert.cc apf::Converter::createRemotes */
static void rebuild_net(struct mds_net* net,
struct mds* m,
struct mds_net* net2,
struct mds* m2,
struct mds_tag* new_of)
{
int d;
mds_id e;
mds_id ne;
mds_id ce;
mds_id nce;
struct mds_copies* cs;
struct mds_copy c;
int i;
PCU_Comm_Begin();
for (d = 0; d <= m->d; ++d)
for (e = mds_begin(m, d); e != MDS_NONE; e = mds_next(m, e)) {
cs = mds_get_copies(net, e);
if (!cs)
continue;
ne = lookup(new_of, e);
for (i = 0; i < cs->n; ++i) {
ce = cs->c[i].e;
PCU_COMM_PACK(cs->c[i].p, ce);
PCU_COMM_PACK(cs->c[i].p, ne);
}
}
PCU_Comm_Send();
while (PCU_Comm_Listen()) {
c.p = PCU_Comm_Sender();
while (!PCU_Comm_Unpacked()) {
PCU_COMM_UNPACK(ce);
PCU_COMM_UNPACK(ne);
c.e = ne;
nce = lookup(new_of, ce);
mds_add_copy(net2, m2, nce, c);
}
}
}
void binGraph::migrate(agi::EdgePartitionMap& map) {
EdgePartitionMap::iterator itr;
PCU_Comm_Begin();
for (itr = map.begin();itr!=map.end();itr++) {
lid_t lid = itr->first;
gid_t v1 = local_unmap[u(lid)];
gid_t v2 = local_unmap[edge_list[0][lid]];
PCU_COMM_PACK(itr->second,v1);
PCU_COMM_PACK(itr->second,v2);
}
PCU_Comm_Send();
std::vector<gid_t> recv_edges;
while (PCU_Comm_Receive()) {
gid_t v1;
PCU_COMM_UNPACK(v1);
recv_edges.push_back(v1);
}
num_global_verts = PCU_Add_Long(num_global_verts);
if (numEdgeTypes()==0)
addEdgeType();
num_local_edges[0] = recv_edges.size()/2;
num_global_edges[0] = PCU_Add_Long(num_local_edges[0]);
if (edge_list[0])
delete edge_list[0];
edge_list[0] = new gid_t[num_local_edges[0]*2];
std::copy(recv_edges.begin(),recv_edges.end(),edge_list[0]);
vtx_mapping.clear();
int32_t* ranks = (int32_t*)malloc(num_global_verts*sizeof(int32_t));
for (int i=0;i<num_global_verts;i++)
ranks[i] = -1;
for (lid_t i=0;i<num_local_edges[0]*2;i++) {
ranks[edge_list[0][i]] = PCU_Comm_Self();
}
create_dist_csr(ranks,0,false);
delete [] ranks;
//TODO: Make much more efficient
PCU_Comm_Begin();
for (int i=0;i<num_local_verts;i++) {
for (int j=1;j<PCU_Comm_Peers();j++)
PCU_COMM_PACK((PCU_Comm_Self()+j)%PCU_Comm_Peers(),local_unmap[i]);
}
PCU_Comm_Send();
std::vector<part_t> owns;
std::vector<gid_t> dups;
degree_list[SPLIT_TYPE] = new lid_t[num_local_verts+1];
for (int i=0;i<num_local_verts+1;++i)
degree_list[SPLIT_TYPE][i]=0;
while (PCU_Comm_Receive()) {
gid_t gid;
PCU_COMM_UNPACK(gid);
map_t::iterator itr = vtx_mapping.find(gid);
if (itr!=vtx_mapping.end()) {
dups.push_back(gid);
owns.push_back(PCU_Comm_Sender());
degree_list[SPLIT_TYPE][itr->second+1]++;
}
}
for (int i=1;i<num_local_verts+1;++i)
degree_list[SPLIT_TYPE][i]+=degree_list[SPLIT_TYPE][i-1];
assert(degree_list[SPLIT_TYPE][num_local_verts] ==dups.size());
num_ghost_verts = dups.size();
num_local_edges[SPLIT_TYPE] = dups.size();
ghost_unmap = new gid_t[dups.size()];
owners = new part_t[dups.size()];
uint64_t* temp_counts = (uint64_t*)malloc(num_local_verts*sizeof(uint64_t));
memcpy(temp_counts, degree_list[SPLIT_TYPE], num_local_verts*sizeof(uint64_t));
edge_list[SPLIT_TYPE] = new lid_t[dups.size()];
for (unsigned int i=0;i<dups.size();i++) {
lid_t lid = vtx_mapping[dups[i]];
edge_list[SPLIT_TYPE][temp_counts[lid]++] = num_local_verts+i;
ghost_unmap[i]=dups[i];
owners[i] = owns[i];
}
num_global_edges[SPLIT_TYPE] = PCU_Add_Long(num_local_edges[SPLIT_TYPE]);
}
