void partitionInfo(agi::binGraph* g) { lid_t total_verts = g->numTotalVtxs(); lid_t global_verts = g->numGlobalVtxs(); lid_t edges = g->numLocalEdges()+g->numLocalEdges(SPLIT_TYPE); lid_t min = PCU_Min_Int(total_verts); lid_t max = PCU_Max_Int(total_verts); lid_t tot = PCU_Add_Long(total_verts); double avg = ((double)tot)/PCU_Comm_Peers(); double imb = max/avg; double inc = ((double)(tot-global_verts))/global_verts*100; if (!PCU_Comm_Self()) printf("Vertices: Min %lu Max %lu Tot %lu Inc %1.4f Avg %1.4f Imb %1.3f\n",min,max,tot,inc,avg,imb); min = PCU_Min_Int(edges); max = PCU_Max_Int(edges); tot = PCU_Add_Long(edges); avg = ((double)tot)/PCU_Comm_Peers(); imb = max/avg; if (!PCU_Comm_Self()) printf("Edges: Min %lu Max %lu Tot %lu Avg %1.4f Imb %1.3f\n",min,max,tot,avg,imb); lid_t edge_cut =0; agi::EdgeIterator* eitr = g->begin(0); for (int i=0;i<g->numLocalEdges();i++) { agi::GraphEdge* e = g->iterate(eitr); if (g->owner(g->v(e))!=PCU_Comm_Self()) edge_cut++; } edge_cut+=g->numLocalEdges(SPLIT_TYPE); edge_cut = PCU_Add_Long(edge_cut); if (!PCU_Comm_Self()) printf("Edge Cut: %lu\n",edge_cut); }
static void note_local_link(mds_id i, struct mds_copy c, void* u) { if (c.p == PCU_Comm_Self()) { if (i < mds_index(c.e)) note_peer(u, PCU_Comm_Self()); else /* hack id to store self-receivers */ note_peer(u, PCU_Comm_Peers()); } }
static void take_local_link(mds_id i, struct mds_copy c, void* u) { struct mds_links* ln = u; int self = find_peer(ln, PCU_Comm_Self()); int other = find_peer(ln, PCU_Comm_Peers()); mds_id j = mds_index(c.e); if ((PCU_Comm_Self() == c.p) && (i < j)) { ln->l[self][ln->n[self]] = i; ln->l[other][ln->n[other]] = j; /* use ns as (redundant) position keepers */ ++ln->n[self]; ++ln->n[other]; } }
void mds_set_type_links(struct mds_net* net, struct mds* m, int t, struct mds_links* ln) { unsigned i; unsigned j; unsigned* in; struct mds_copy c; PCU_Comm_Begin(); for (i = 0; i < ln->np; ++i) { PCU_ALWAYS_ASSERT(ln->l); for (j = 0; j < ln->n[i]; ++j) PCU_COMM_PACK(ln->p[i], ln->l[i][j]); } PCU_Comm_Send(); while (PCU_Comm_Listen()) { c.p = PCU_Comm_Sender(); PCU_ALWAYS_ASSERT(c.p != PCU_Comm_Self()); i = find_peer(ln, c.p); in = PCU_Comm_Extract(ln->n[i] * sizeof(unsigned)); for (j = 0; j < ln->n[i]; ++j) { c.e = mds_identify(t, in[j]); mds_add_copy(net, m, mds_identify(t, ln->l[i][j]), c); } } }
static void send_remote_link(mds_id i, struct mds_copy c, void* u) { (void)i; (void)u; if (PCU_Comm_Self() < c.p) PCU_COMM_PACK(c.p, c.e); }
static void switchToOriginals(int npartitions) { int self = PCU_Comm_Self(); int groupRank = self / npartitions; int group = self % npartitions; MPI_Comm groupComm; MPI_Comm_split(MPI_COMM_WORLD, group, groupRank, &groupComm); PCU_Switch_Comm(groupComm); }
static void take_remote_link(mds_id i, struct mds_copy c, void* u) { struct mds_links* ln = u; int pi; if (PCU_Comm_Self() < c.p) { pi = find_peer(ln, c.p); ln->l[pi][ln->n[pi]] = i; /* use n as position keeper */ ++ln->n[pi]; } }
static int owns_copies(mds_id e, struct mds_copies* c) { int i; struct mds_copy mc; mc.e = e; mc.p = PCU_Comm_Self(); for (i = 0; i < c->n; ++i) if (copy_less(c->c[i], mc)) return 0; return 1; }
void mds_free_local_links(struct mds_links* ln) { int self, other; self = find_peer(ln, PCU_Comm_Self()); if (self == -1) return; other = find_peer(ln, PCU_Comm_Peers()); ln->n[self] = ln->n[other] = 0; free(ln->l[self]); free(ln->l[other]); ln->l[self] = ln->l[other] = NULL; }
static int read_magic_number(FILE* f) { int magic; if (!seek_after_header(f, magic_name)) { if (!PCU_Comm_Self()) fprintf(stderr,"warning: not swapping bytes\n"); rewind(f); return 0; } my_fread(&magic, sizeof(int), 1, f); return magic != MAGIC; }
//TODO: optimize these operations using PCU //Private Functions etype binGraph::load_edges(char *filename, uint64_t*& read_edges, uint64_t& m_read) { FILE *infp = fopen(filename, "rb"); fseek(infp, 0L, SEEK_END); uint64_t file_size = ftell(infp); fseek(infp, 0L, SEEK_SET); etype t = addEdgeType(); num_global_edges[t] = file_size/(2*sizeof(uint32_t)); uint64_t read_offset_start = PCU_Comm_Self()*2*sizeof(uint32_t)* (num_global_edges[t] / (uint64_t)PCU_Comm_Peers()); uint64_t read_offset_end = (PCU_Comm_Self()+1)*2*sizeof(uint32_t)* (num_global_edges[t] / (uint64_t)PCU_Comm_Peers()); if (PCU_Comm_Self() == PCU_Comm_Peers() - 1) read_offset_end = 2*sizeof(uint32_t)*num_global_edges[t]; m_read = (read_offset_end - read_offset_start)/(2*sizeof(uint32_t)); uint32_t* temp_read = (uint32_t*)malloc(2*m_read*sizeof(uint32_t)); read_edges = (uint64_t*)malloc(2*m_read*sizeof(uint64_t)); fseek(infp, read_offset_start, SEEK_SET); fread(temp_read, m_read, 2*sizeof(uint32_t), infp); fclose(infp); for (uint64_t i = 0; i < m_read*2; ++i) read_edges[i] = (uint64_t)temp_read[i]; free(temp_read); num_global_verts = 0; for (uint64_t i = 0; i < m_read*2; ++i) if (read_edges[i] > num_global_verts) { num_global_verts = read_edges[i]; } MPI_Allreduce(MPI_IN_PLACE, &num_global_verts, 1, MPI_UINT64_T, MPI_MAX, PCU_Get_Comm()); num_global_verts += 1; return t; }
void mds_set_local_matches(struct mds_net* net, struct mds* m, int t, struct mds_links* ln) { int self, other; unsigned i; mds_id a, b; struct mds_copy c; c.p = PCU_Comm_Self(); self = find_peer(ln, PCU_Comm_Self()); if (self == -1) return; other = find_peer(ln, PCU_Comm_Peers()); assert(ln->n[self] = ln->n[other]); for (i = 0; i < ln->n[self]; ++i) { a = mds_identify(t, ln->l[self][i]); b = mds_identify(t, ln->l[other][i]); c.e = b; mds_add_copy(net, m, a, c); c.e = a; mds_add_copy(net, m, b, c); } }
void mds_get_local_matches(struct mds_net* net, struct mds* m, int t, struct mds_links* ln) { int self, other; for_type_net(net, m, t, note_local_link, ln); self = find_peer(ln, PCU_Comm_Self()); if (self == -1) return; other = find_peer(ln, PCU_Comm_Peers()); assert(ln->n[self] == ln->n[other]); ln->l[self] = malloc(ln->n[self] * sizeof(unsigned)); ln->l[other] = malloc(ln->n[other] * sizeof(unsigned)); ln->n[self] = 0; ln->n[other] = 0; for_type_net(net, m, t, take_local_link, ln); }
static int find_header(FILE* f, const char* name, char header[PH_LINE]) { char* hname; long bytes; char tmp[PH_LINE]; while (fgets(header, PH_LINE, f)) { if ((header[0] == '#') || (header[0] == '\n')) continue; strncpy(tmp, header, PH_LINE-1); tmp[PH_LINE-1] = '\0'; parse_header(tmp, &hname, &bytes, 0, NULL); if (!strncmp(name, hname, strlen(name))) return 1; fseek(f, bytes, SEEK_CUR); } if (!PCU_Comm_Self()) fprintf(stderr,"warning: phIO could not find \"%s\"\n",name); return 0; }
/* algorithm courtesy of Sebastian Rettenberger: use brokers/routers for the vertex global ids. Although we have used this trick before (see mpas/apfMPAS.cc), I didn't think to use it here, so credit is given. */ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert) { Gid max = getMax(globalToVert); Gid total = max + 1; int peers = PCU_Comm_Peers(); int quotient = total / peers; int remainder = total % peers; int mySize = quotient; int self = PCU_Comm_Self(); if (self == (peers - 1)) mySize += remainder; typedef std::vector< std::vector<int> > TmpParts; TmpParts tmpParts(mySize); /* if we have a vertex, send its global id to the broker for that global id */ PCU_Comm_Begin(); APF_ITERATE(GlobalToVert, globalToVert, it) { int gid = it->first; int to = std::min(peers - 1, gid / quotient); PCU_COMM_PACK(to, gid); }
void mds_get_type_links(struct mds_net* net, struct mds* m, int t, struct mds_links* ln) { unsigned i; /* count remote links in np, p and n */ for_type_net(net, m, t, note_remote_link, ln); alloc_links(ln); for (i = 0; i < ln->np; ++i) if (((unsigned)PCU_Comm_Self()) < ln->p[i]) /* zero n's for behavior in take_copy */ ln->n[i] = 0; /* record indices in local order for owned boundaries */ for_type_net(net, m, t, take_remote_link, ln); PCU_Comm_Begin(); /* send indices in local order for owned boundaries */ for_type_net(net, m, t, send_remote_link, NULL); PCU_Comm_Send(); while (PCU_Comm_Listen()) /* recv indices in remote order for non-owned boundaries */ recv_links(ln); }
Albany::PUMIMeshStruct::PUMIMeshStruct( const Teuchos::RCP<Teuchos::ParameterList>& params, const Teuchos::RCP<const Teuchos_Comm>& commT) { params->validateParameters( *(PUMIMeshStruct::getValidDiscretizationParameters()), 0); outputFileName = params->get<std::string>("PUMI Output File Name", ""); outputInterval = params->get<int>("PUMI Write Interval", 1); // write every time step default std::string model_file; if(params->isParameter("Mesh Model Input File Name")) model_file = params->get<std::string>("Mesh Model Input File Name"); #ifdef SCOREC_SIMMODEL if (params->isParameter("Acis Model Input File Name")) model_file = params->get<std::string>("Parasolid Model Input File Name"); if(params->isParameter("Parasolid Model Input File Name")) model_file = params->get<std::string>("Parasolid Model Input File Name"); #endif if (params->isParameter("PUMI Input File Name")) { std::string mesh_file = params->get<std::string>("PUMI Input File Name"); mesh = 0; // If we are running in parallel but have a single mesh file, split it and rebalance bool useSerialMesh = params->get<bool>("Use Serial Mesh", false); if (useSerialMesh && commT->getSize() > 1){ // do the equivalent of the SCOREC "split" utility apf::Migration* plan = 0; gmi_model* g = 0; g = gmi_load(model_file.c_str()); bool isOriginal = ((PCU_Comm_Self() % commT->getSize()) == 0); switchToOriginals(commT->getSize()); if (isOriginal) { mesh = apf::loadMdsMesh(g, mesh_file.c_str()); plan = getPlan(mesh, commT->getSize()); } switchToAll(); mesh = repeatMdsMesh(mesh, g, plan, commT->getSize()); } else { mesh = apf::loadMdsMesh(model_file.c_str(), mesh_file.c_str()); } } else { int nex = params->get<int>("1D Elements", 0); int ney = params->get<int>("2D Elements", 0); int nez = params->get<int>("3D Elements", 0); double wx = params->get<double>("1D Scale", 1); double wy = params->get<double>("2D Scale", 1); double wz = params->get<double>("3D Scale", 1); bool is = ! params->get<bool>("Hexahedral", true); buildBoxMesh(nex, ney, nez, wx, wy, wz, is); } model = mesh->getModel(); // Tell the mesh that we'll handle deleting the model. apf::disownMdsModel(mesh); bool isQuadMesh = params->get<bool>("2nd Order Mesh",false); if (isQuadMesh) apf::changeMeshShape(mesh, apf::getSerendipity(), false); // Resize mesh after input if indicated in the input file // User has indicated a desired element size in input file if(params->isParameter("Resize Input Mesh Element Size")){ SizeFunction sizeFunction(params->get<double>( "Resize Input Mesh Element Size", 0.1)); int num_iters = params->get<int>( "Max Number of Mesh Adapt Iterations", 1); ma::Input* input = ma::configure(mesh,&sizeFunction); input->maximumIterations = num_iters; input->shouldSnap = false; ma::adapt(input); } // get the continuation step to write a restart file restartWriteStep = params->get<int>("Write Restart File at Step",0); APFMeshStruct::init(params, commT); // if we have a restart time, we will want to override some of // the default paramaters set by APFMeshStruct::init if (params->isParameter("PUMI Restart Time")) { hasRestartSolution = true; restartDataTime = params->get<double>("PUMI Restart Time", 0.0); std::string name = params->get<std::string>("PUMI Input File Name"); if (!PCU_Comm_Self()) std::cout << "Restarting from time: " << restartDataTime << " from restart file: " << name << std::endl; } if (params->isParameter("Load FELIX Data")) shouldLoadFELIXData = true; }
void binGraph::migrate(agi::EdgePartitionMap& map) { EdgePartitionMap::iterator itr; PCU_Comm_Begin(); for (itr = map.begin();itr!=map.end();itr++) { lid_t lid = itr->first; gid_t v1 = local_unmap[u(lid)]; gid_t v2 = local_unmap[edge_list[0][lid]]; PCU_COMM_PACK(itr->second,v1); PCU_COMM_PACK(itr->second,v2); } PCU_Comm_Send(); std::vector<gid_t> recv_edges; while (PCU_Comm_Receive()) { gid_t v1; PCU_COMM_UNPACK(v1); recv_edges.push_back(v1); } num_global_verts = PCU_Add_Long(num_global_verts); if (numEdgeTypes()==0) addEdgeType(); num_local_edges[0] = recv_edges.size()/2; num_global_edges[0] = PCU_Add_Long(num_local_edges[0]); if (edge_list[0]) delete edge_list[0]; edge_list[0] = new gid_t[num_local_edges[0]*2]; std::copy(recv_edges.begin(),recv_edges.end(),edge_list[0]); vtx_mapping.clear(); int32_t* ranks = (int32_t*)malloc(num_global_verts*sizeof(int32_t)); for (int i=0;i<num_global_verts;i++) ranks[i] = -1; for (lid_t i=0;i<num_local_edges[0]*2;i++) { ranks[edge_list[0][i]] = PCU_Comm_Self(); } create_dist_csr(ranks,0,false); delete [] ranks; //TODO: Make much more efficient PCU_Comm_Begin(); for (int i=0;i<num_local_verts;i++) { for (int j=1;j<PCU_Comm_Peers();j++) PCU_COMM_PACK((PCU_Comm_Self()+j)%PCU_Comm_Peers(),local_unmap[i]); } PCU_Comm_Send(); std::vector<part_t> owns; std::vector<gid_t> dups; degree_list[SPLIT_TYPE] = new lid_t[num_local_verts+1]; for (int i=0;i<num_local_verts+1;++i) degree_list[SPLIT_TYPE][i]=0; while (PCU_Comm_Receive()) { gid_t gid; PCU_COMM_UNPACK(gid); map_t::iterator itr = vtx_mapping.find(gid); if (itr!=vtx_mapping.end()) { dups.push_back(gid); owns.push_back(PCU_Comm_Sender()); degree_list[SPLIT_TYPE][itr->second+1]++; } } for (int i=1;i<num_local_verts+1;++i) degree_list[SPLIT_TYPE][i]+=degree_list[SPLIT_TYPE][i-1]; assert(degree_list[SPLIT_TYPE][num_local_verts] ==dups.size()); num_ghost_verts = dups.size(); num_local_edges[SPLIT_TYPE] = dups.size(); ghost_unmap = new gid_t[dups.size()]; owners = new part_t[dups.size()]; uint64_t* temp_counts = (uint64_t*)malloc(num_local_verts*sizeof(uint64_t)); memcpy(temp_counts, degree_list[SPLIT_TYPE], num_local_verts*sizeof(uint64_t)); edge_list[SPLIT_TYPE] = new lid_t[dups.size()]; for (unsigned int i=0;i<dups.size();i++) { lid_t lid = vtx_mapping[dups[i]]; edge_list[SPLIT_TYPE][temp_counts[lid]++] = num_local_verts+i; ghost_unmap[i]=dups[i]; owners[i] = owns[i]; } num_global_edges[SPLIT_TYPE] = PCU_Add_Long(num_local_edges[SPLIT_TYPE]); }
void phi() { if (!PCU_Comm_Self()) { printf("hi\n"); } }
int binGraph::create_dist_csr(int32_t* ranks,etype t,bool createGhost) { num_local_verts = 0; for (uint64_t i = 0; i < num_global_verts; ++i) if (ranks[i] == PCU_Comm_Self()) ++num_local_verts; local_unmap = (uint64_t*)malloc(num_local_verts*sizeof(uint64_t)); uint64_t cur_label = 0; for (uint64_t i = 0; i < num_local_edges[t]*2; i++) { uint64_t out = edge_list[t][i]; if (ranks[out] != PCU_Comm_Self()) continue; if (vtx_mapping.count(out) == 0) { vtx_mapping[out] = cur_label; local_unmap[cur_label] = out; edge_list[t][i] = cur_label++; } else edge_list[t][i] = vtx_mapping[out]; } uint64_t* tmp_edges = (uint64_t*)malloc(num_local_edges[t]*sizeof(uint64_t)); uint64_t* temp_counts = (uint64_t*)malloc(num_local_verts*sizeof(uint64_t)); degree_list[t] = (uint64_t*)malloc((num_local_verts+1)*sizeof(uint64_t)); for (uint64_t i = 0; i < num_local_verts+1; ++i) degree_list[t][i] = 0; for (uint64_t i = 0; i < num_local_verts; ++i) temp_counts[i] = 0; for (uint64_t i = 0; i < num_local_edges[t]*2; i+=2) ++temp_counts[edge_list[t][i]]; for (uint64_t i = 0; i < num_local_verts; ++i) degree_list[t][i+1] = degree_list[t][i] + temp_counts[i]; memcpy(temp_counts, degree_list[t], num_local_verts*sizeof(uint64_t)); for (uint64_t i = 0; i < num_local_edges[t]*2; i+=2) tmp_edges[temp_counts[edge_list[t][i]]++] = edge_list[t][i+1]; free(edge_list[t]); edge_list[t] = tmp_edges; if (createGhost) { cur_label = num_local_verts; std::vector<uint64_t> nonlocal_vids; for (uint64_t i = 0; i < num_local_edges[t]; ++i) { uint64_t out = edge_list[t][i]; if (vtx_mapping.find(out) ==vtx_mapping.end()) { vtx_mapping[out] = cur_label; edge_list[t][i] = cur_label++; nonlocal_vids.push_back(out); } else edge_list[t][i] = vtx_mapping[out]; } num_ghost_verts = cur_label - num_local_verts; if (num_ghost_verts>0) { ghost_unmap = (uint64_t*)malloc(num_ghost_verts*sizeof(uint64_t)); owners = (int32_t*)malloc(num_ghost_verts*sizeof(int32_t)); for (uint64_t i = 0; i < nonlocal_vids.size(); ++i) { ghost_unmap[i] = nonlocal_vids[i]; owners[i] = ranks[nonlocal_vids[i]]; } } } return 0; }
int main(int argc, char* argv[]) { MPI_Init(&argc,&argv); PCU_Comm_Init(); PCU_Debug_Open(); if ( argc != 2&&argc!=3 ) { if ( !PCU_Comm_Self() ) printf("Usage: %s <binary_graph_file> [vertex_partition_file]",argv[0]); PCU_Comm_Free(); MPI_Finalize(); assert(false); } agi::binGraph* g; zagi::ZoltanCutVertex* ptn; int self = PCU_Comm_Self(); int num_parts = PCU_Comm_Peers(); PCU_Switch_Comm(MPI_COMM_SELF); if (!self) { g = new agi::binGraph(argv[1]); ptn = new zagi::ZoltanCutVertex(g,num_parts); ptn->run(); } else g = new agi::binGraph(); agi::EdgePartitionMap map; if (!self) { ptn->createPtn(map); delete ptn; } PCU_Switch_Comm(MPI_COMM_WORLD); g->migrate(map); partitionInfo(g); delete g; if (!PCU_Comm_Self()) printf("Block Partitioning\n"); g = new agi::binGraph(argv[1]); partitionInfo(g); delete g; if (argc==3) { if (!PCU_Comm_Self()) printf("Partitioned: %s\n",argv[2]); g = new agi::binGraph(argv[1],argv[2]); partitionInfo(g); delete g; } PCU_Barrier(); if (!PCU_Comm_Self()) printf("\nAll tests passed\n"); PCU_Comm_Free(); MPI_Finalize(); }
static void note_remote_link(mds_id i, struct mds_copy c, void* u) { (void)i; if (c.p != PCU_Comm_Self()) note_peer(u, c.p); }