void partitionInfo(agi::binGraph* g) {
lid_t total_verts = g->numTotalVtxs();
lid_t global_verts = g->numGlobalVtxs();
lid_t edges = g->numLocalEdges()+g->numLocalEdges(SPLIT_TYPE);
lid_t min = PCU_Min_Int(total_verts);
lid_t max = PCU_Max_Int(total_verts);
lid_t tot = PCU_Add_Long(total_verts);
double avg = ((double)tot)/PCU_Comm_Peers();
double imb = max/avg;
double inc = ((double)(tot-global_verts))/global_verts*100;
if (!PCU_Comm_Self())
printf("Vertices: Min %lu Max %lu Tot %lu Inc %1.4f Avg %1.4f Imb %1.3f\n",min,max,tot,inc,avg,imb);
min = PCU_Min_Int(edges);
max = PCU_Max_Int(edges);
tot = PCU_Add_Long(edges);
avg = ((double)tot)/PCU_Comm_Peers();
imb = max/avg;
if (!PCU_Comm_Self())
printf("Edges: Min %lu Max %lu Tot %lu Avg %1.4f Imb %1.3f\n",min,max,tot,avg,imb);
lid_t edge_cut =0;
agi::EdgeIterator* eitr = g->begin(0);
for (int i=0;i<g->numLocalEdges();i++) {
agi::GraphEdge* e = g->iterate(eitr);
if (g->owner(g->v(e))!=PCU_Comm_Self())
edge_cut++;
}
edge_cut+=g->numLocalEdges(SPLIT_TYPE);
edge_cut = PCU_Add_Long(edge_cut);
if (!PCU_Comm_Self())
printf("Edge Cut: %lu\n",edge_cut);
}
static void note_local_link(mds_id i, struct mds_copy c, void* u)
{
if (c.p == PCU_Comm_Self()) {
if (i < mds_index(c.e))
note_peer(u, PCU_Comm_Self());
else /* hack id to store self-receivers */
note_peer(u, PCU_Comm_Peers());
}
}
static void take_local_link(mds_id i, struct mds_copy c, void* u)
{
struct mds_links* ln = u;
int self = find_peer(ln, PCU_Comm_Self());
int other = find_peer(ln, PCU_Comm_Peers());
mds_id j = mds_index(c.e);
if ((PCU_Comm_Self() == c.p) && (i < j)) {
ln->l[self][ln->n[self]] = i;
ln->l[other][ln->n[other]] = j;
/* use ns as (redundant) position keepers */
++ln->n[self];
++ln->n[other];
}
}
void mds_set_type_links(struct mds_net* net, struct mds* m,
int t, struct mds_links* ln)
{
unsigned i;
unsigned j;
unsigned* in;
struct mds_copy c;
PCU_Comm_Begin();
for (i = 0; i < ln->np; ++i) {
PCU_ALWAYS_ASSERT(ln->l);
for (j = 0; j < ln->n[i]; ++j)
PCU_COMM_PACK(ln->p[i], ln->l[i][j]);
}
PCU_Comm_Send();
while (PCU_Comm_Listen()) {
c.p = PCU_Comm_Sender();
PCU_ALWAYS_ASSERT(c.p != PCU_Comm_Self());
i = find_peer(ln, c.p);
in = PCU_Comm_Extract(ln->n[i] * sizeof(unsigned));
for (j = 0; j < ln->n[i]; ++j) {
c.e = mds_identify(t, in[j]);
mds_add_copy(net, m, mds_identify(t, ln->l[i][j]), c);
}
}
}
static void send_remote_link(mds_id i, struct mds_copy c, void* u)
{
(void)i;
(void)u;
if (PCU_Comm_Self() < c.p)
PCU_COMM_PACK(c.p, c.e);
}
static void switchToOriginals(int npartitions)
{
int self = PCU_Comm_Self();
int groupRank = self / npartitions;
int group = self % npartitions;
MPI_Comm groupComm;
MPI_Comm_split(MPI_COMM_WORLD, group, groupRank, &groupComm);
PCU_Switch_Comm(groupComm);
}
static void take_remote_link(mds_id i, struct mds_copy c, void* u)
{
struct mds_links* ln = u;
int pi;
if (PCU_Comm_Self() < c.p) {
pi = find_peer(ln, c.p);
ln->l[pi][ln->n[pi]] = i;
/* use n as position keeper */
++ln->n[pi];
}
}
static int owns_copies(mds_id e, struct mds_copies* c)
{
int i;
struct mds_copy mc;
mc.e = e;
mc.p = PCU_Comm_Self();
for (i = 0; i < c->n; ++i)
if (copy_less(c->c[i], mc))
return 0;
return 1;
}
void mds_free_local_links(struct mds_links* ln)
{
int self, other;
self = find_peer(ln, PCU_Comm_Self());
if (self == -1)
return;
other = find_peer(ln, PCU_Comm_Peers());
ln->n[self] = ln->n[other] = 0;
free(ln->l[self]);
free(ln->l[other]);
ln->l[self] = ln->l[other] = NULL;
}
static int read_magic_number(FILE* f)
{
int magic;
if (!seek_after_header(f, magic_name)) {
if (!PCU_Comm_Self())
fprintf(stderr,"warning: not swapping bytes\n");
rewind(f);
return 0;
}
my_fread(&magic, sizeof(int), 1, f);
return magic != MAGIC;
}
//TODO: optimize these operations using PCU
//Private Functions
etype binGraph::load_edges(char *filename, uint64_t*& read_edges,
uint64_t& m_read) {
FILE *infp = fopen(filename, "rb");
fseek(infp, 0L, SEEK_END);
uint64_t file_size = ftell(infp);
fseek(infp, 0L, SEEK_SET);
etype t = addEdgeType();
num_global_edges[t] = file_size/(2*sizeof(uint32_t));
uint64_t read_offset_start = PCU_Comm_Self()*2*sizeof(uint32_t)*
(num_global_edges[t] / (uint64_t)PCU_Comm_Peers());
uint64_t read_offset_end = (PCU_Comm_Self()+1)*2*sizeof(uint32_t)*
(num_global_edges[t] / (uint64_t)PCU_Comm_Peers());
if (PCU_Comm_Self() == PCU_Comm_Peers() - 1)
read_offset_end = 2*sizeof(uint32_t)*num_global_edges[t];
m_read = (read_offset_end - read_offset_start)/(2*sizeof(uint32_t));
uint32_t* temp_read = (uint32_t*)malloc(2*m_read*sizeof(uint32_t));
read_edges = (uint64_t*)malloc(2*m_read*sizeof(uint64_t));
fseek(infp, read_offset_start, SEEK_SET);
fread(temp_read, m_read, 2*sizeof(uint32_t), infp);
fclose(infp);
for (uint64_t i = 0; i < m_read*2; ++i)
read_edges[i] = (uint64_t)temp_read[i];
free(temp_read);
num_global_verts = 0;
for (uint64_t i = 0; i < m_read*2; ++i)
if (read_edges[i] > num_global_verts) {
num_global_verts = read_edges[i];
}
MPI_Allreduce(MPI_IN_PLACE, &num_global_verts, 1, MPI_UINT64_T, MPI_MAX, PCU_Get_Comm());
num_global_verts += 1;
return t;
}
void mds_set_local_matches(struct mds_net* net, struct mds* m,
int t, struct mds_links* ln)
{
int self, other;
unsigned i;
mds_id a, b;
struct mds_copy c;
c.p = PCU_Comm_Self();
self = find_peer(ln, PCU_Comm_Self());
if (self == -1)
return;
other = find_peer(ln, PCU_Comm_Peers());
assert(ln->n[self] = ln->n[other]);
for (i = 0; i < ln->n[self]; ++i) {
a = mds_identify(t, ln->l[self][i]);
b = mds_identify(t, ln->l[other][i]);
c.e = b;
mds_add_copy(net, m, a, c);
c.e = a;
mds_add_copy(net, m, b, c);
}
}
void mds_get_local_matches(struct mds_net* net, struct mds* m,
int t, struct mds_links* ln)
{
int self, other;
for_type_net(net, m, t, note_local_link, ln);
self = find_peer(ln, PCU_Comm_Self());
if (self == -1)
return;
other = find_peer(ln, PCU_Comm_Peers());
assert(ln->n[self] == ln->n[other]);
ln->l[self] = malloc(ln->n[self] * sizeof(unsigned));
ln->l[other] = malloc(ln->n[other] * sizeof(unsigned));
ln->n[self] = 0;
ln->n[other] = 0;
for_type_net(net, m, t, take_local_link, ln);
}
static int find_header(FILE* f, const char* name, char header[PH_LINE])
{
char* hname;
long bytes;
char tmp[PH_LINE];
while (fgets(header, PH_LINE, f)) {
if ((header[0] == '#') || (header[0] == '\n'))
continue;
strncpy(tmp, header, PH_LINE-1);
tmp[PH_LINE-1] = '\0';
parse_header(tmp, &hname, &bytes, 0, NULL);
if (!strncmp(name, hname, strlen(name)))
return 1;
fseek(f, bytes, SEEK_CUR);
}
if (!PCU_Comm_Self())
fprintf(stderr,"warning: phIO could not find \"%s\"\n",name);
return 0;
}
/* algorithm courtesy of Sebastian Rettenberger:
use brokers/routers for the vertex global ids.
Although we have used this trick before (see mpas/apfMPAS.cc),
I didn't think to use it here, so credit is given. */
static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
{
Gid max = getMax(globalToVert);
Gid total = max + 1;
int peers = PCU_Comm_Peers();
int quotient = total / peers;
int remainder = total % peers;
int mySize = quotient;
int self = PCU_Comm_Self();
if (self == (peers - 1))
mySize += remainder;
typedef std::vector< std::vector<int> > TmpParts;
TmpParts tmpParts(mySize);
/* if we have a vertex, send its global id to the
broker for that global id */
PCU_Comm_Begin();
APF_ITERATE(GlobalToVert, globalToVert, it) {
int gid = it->first;
int to = std::min(peers - 1, gid / quotient);
PCU_COMM_PACK(to, gid);
}
void mds_get_type_links(struct mds_net* net, struct mds* m,
int t, struct mds_links* ln)
{
unsigned i;
/* count remote links in np, p and n */
for_type_net(net, m, t, note_remote_link, ln);
alloc_links(ln);
for (i = 0; i < ln->np; ++i)
if (((unsigned)PCU_Comm_Self()) < ln->p[i])
/* zero n's for behavior in take_copy */
ln->n[i] = 0;
/* record indices in local order for owned boundaries */
for_type_net(net, m, t, take_remote_link, ln);
PCU_Comm_Begin();
/* send indices in local order for owned boundaries */
for_type_net(net, m, t, send_remote_link, NULL);
PCU_Comm_Send();
while (PCU_Comm_Listen())
/* recv indices in remote order for non-owned boundaries */
recv_links(ln);
}
Albany::PUMIMeshStruct::PUMIMeshStruct(
const Teuchos::RCP<Teuchos::ParameterList>& params,
const Teuchos::RCP<const Teuchos_Comm>& commT)
{
params->validateParameters(
*(PUMIMeshStruct::getValidDiscretizationParameters()), 0);
outputFileName = params->get<std::string>("PUMI Output File Name", "");
outputInterval = params->get<int>("PUMI Write Interval", 1); // write every time step default
std::string model_file;
if(params->isParameter("Mesh Model Input File Name"))
model_file = params->get<std::string>("Mesh Model Input File Name");
#ifdef SCOREC_SIMMODEL
if (params->isParameter("Acis Model Input File Name"))
model_file = params->get<std::string>("Parasolid Model Input File Name");
if(params->isParameter("Parasolid Model Input File Name"))
model_file = params->get<std::string>("Parasolid Model Input File Name");
#endif
if (params->isParameter("PUMI Input File Name")) {
std::string mesh_file = params->get<std::string>("PUMI Input File Name");
mesh = 0;
// If we are running in parallel but have a single mesh file, split it and rebalance
bool useSerialMesh = params->get<bool>("Use Serial Mesh", false);
if (useSerialMesh && commT->getSize() > 1){ // do the equivalent of the SCOREC "split" utility
apf::Migration* plan = 0;
gmi_model* g = 0;
g = gmi_load(model_file.c_str());
bool isOriginal = ((PCU_Comm_Self() % commT->getSize()) == 0);
switchToOriginals(commT->getSize());
if (isOriginal) {
mesh = apf::loadMdsMesh(g, mesh_file.c_str());
plan = getPlan(mesh, commT->getSize());
}
switchToAll();
mesh = repeatMdsMesh(mesh, g, plan, commT->getSize());
}
else {
mesh = apf::loadMdsMesh(model_file.c_str(), mesh_file.c_str());
}
} else {
int nex = params->get<int>("1D Elements", 0);
int ney = params->get<int>("2D Elements", 0);
int nez = params->get<int>("3D Elements", 0);
double wx = params->get<double>("1D Scale", 1);
double wy = params->get<double>("2D Scale", 1);
double wz = params->get<double>("3D Scale", 1);
bool is = ! params->get<bool>("Hexahedral", true);
buildBoxMesh(nex, ney, nez, wx, wy, wz, is);
}
model = mesh->getModel();
// Tell the mesh that we'll handle deleting the model.
apf::disownMdsModel(mesh);
bool isQuadMesh = params->get<bool>("2nd Order Mesh",false);
if (isQuadMesh)
apf::changeMeshShape(mesh, apf::getSerendipity(), false);
// Resize mesh after input if indicated in the input file
// User has indicated a desired element size in input file
if(params->isParameter("Resize Input Mesh Element Size")){
SizeFunction sizeFunction(params->get<double>(
"Resize Input Mesh Element Size", 0.1));
int num_iters = params->get<int>(
"Max Number of Mesh Adapt Iterations", 1);
ma::Input* input = ma::configure(mesh,&sizeFunction);
input->maximumIterations = num_iters;
input->shouldSnap = false;
ma::adapt(input);
}
// get the continuation step to write a restart file
restartWriteStep = params->get<int>("Write Restart File at Step",0);
APFMeshStruct::init(params, commT);
// if we have a restart time, we will want to override some of
// the default paramaters set by APFMeshStruct::init
if (params->isParameter("PUMI Restart Time")) {
hasRestartSolution = true;
restartDataTime = params->get<double>("PUMI Restart Time", 0.0);
std::string name = params->get<std::string>("PUMI Input File Name");
if (!PCU_Comm_Self())
std::cout << "Restarting from time: " << restartDataTime
<< " from restart file: " << name << std::endl;
}
if (params->isParameter("Load FELIX Data"))
shouldLoadFELIXData = true;
}
void binGraph::migrate(agi::EdgePartitionMap& map) {
EdgePartitionMap::iterator itr;
PCU_Comm_Begin();
for (itr = map.begin();itr!=map.end();itr++) {
lid_t lid = itr->first;
gid_t v1 = local_unmap[u(lid)];
gid_t v2 = local_unmap[edge_list[0][lid]];
PCU_COMM_PACK(itr->second,v1);
PCU_COMM_PACK(itr->second,v2);
}
PCU_Comm_Send();
std::vector<gid_t> recv_edges;
while (PCU_Comm_Receive()) {
gid_t v1;
PCU_COMM_UNPACK(v1);
recv_edges.push_back(v1);
}
num_global_verts = PCU_Add_Long(num_global_verts);
if (numEdgeTypes()==0)
addEdgeType();
num_local_edges[0] = recv_edges.size()/2;
num_global_edges[0] = PCU_Add_Long(num_local_edges[0]);
if (edge_list[0])
delete edge_list[0];
edge_list[0] = new gid_t[num_local_edges[0]*2];
std::copy(recv_edges.begin(),recv_edges.end(),edge_list[0]);
vtx_mapping.clear();
int32_t* ranks = (int32_t*)malloc(num_global_verts*sizeof(int32_t));
for (int i=0;i<num_global_verts;i++)
ranks[i] = -1;
for (lid_t i=0;i<num_local_edges[0]*2;i++) {
ranks[edge_list[0][i]] = PCU_Comm_Self();
}
create_dist_csr(ranks,0,false);
delete [] ranks;
//TODO: Make much more efficient
PCU_Comm_Begin();
for (int i=0;i<num_local_verts;i++) {
for (int j=1;j<PCU_Comm_Peers();j++)
PCU_COMM_PACK((PCU_Comm_Self()+j)%PCU_Comm_Peers(),local_unmap[i]);
}
PCU_Comm_Send();
std::vector<part_t> owns;
std::vector<gid_t> dups;
degree_list[SPLIT_TYPE] = new lid_t[num_local_verts+1];
for (int i=0;i<num_local_verts+1;++i)
degree_list[SPLIT_TYPE][i]=0;
while (PCU_Comm_Receive()) {
gid_t gid;
PCU_COMM_UNPACK(gid);
map_t::iterator itr = vtx_mapping.find(gid);
if (itr!=vtx_mapping.end()) {
dups.push_back(gid);
owns.push_back(PCU_Comm_Sender());
degree_list[SPLIT_TYPE][itr->second+1]++;
}
}
for (int i=1;i<num_local_verts+1;++i)
degree_list[SPLIT_TYPE][i]+=degree_list[SPLIT_TYPE][i-1];
assert(degree_list[SPLIT_TYPE][num_local_verts] ==dups.size());
num_ghost_verts = dups.size();
num_local_edges[SPLIT_TYPE] = dups.size();
ghost_unmap = new gid_t[dups.size()];
owners = new part_t[dups.size()];
uint64_t* temp_counts = (uint64_t*)malloc(num_local_verts*sizeof(uint64_t));
memcpy(temp_counts, degree_list[SPLIT_TYPE], num_local_verts*sizeof(uint64_t));
edge_list[SPLIT_TYPE] = new lid_t[dups.size()];
for (unsigned int i=0;i<dups.size();i++) {
lid_t lid = vtx_mapping[dups[i]];
edge_list[SPLIT_TYPE][temp_counts[lid]++] = num_local_verts+i;
ghost_unmap[i]=dups[i];
owners[i] = owns[i];
}
num_global_edges[SPLIT_TYPE] = PCU_Add_Long(num_local_edges[SPLIT_TYPE]);
}
void phi() {
if (!PCU_Comm_Self()) {
printf("hi\n");
}
}
int binGraph::create_dist_csr(int32_t* ranks,etype t,bool createGhost)
{
num_local_verts = 0;
for (uint64_t i = 0; i < num_global_verts; ++i)
if (ranks[i] == PCU_Comm_Self())
++num_local_verts;
local_unmap = (uint64_t*)malloc(num_local_verts*sizeof(uint64_t));
uint64_t cur_label = 0;
for (uint64_t i = 0; i < num_local_edges[t]*2; i++) {
uint64_t out = edge_list[t][i];
if (ranks[out] != PCU_Comm_Self())
continue;
if (vtx_mapping.count(out) == 0) {
vtx_mapping[out] = cur_label;
local_unmap[cur_label] = out;
edge_list[t][i] = cur_label++;
}
else
edge_list[t][i] = vtx_mapping[out];
}
uint64_t* tmp_edges = (uint64_t*)malloc(num_local_edges[t]*sizeof(uint64_t));
uint64_t* temp_counts = (uint64_t*)malloc(num_local_verts*sizeof(uint64_t));
degree_list[t] = (uint64_t*)malloc((num_local_verts+1)*sizeof(uint64_t));
for (uint64_t i = 0; i < num_local_verts+1; ++i)
degree_list[t][i] = 0;
for (uint64_t i = 0; i < num_local_verts; ++i)
temp_counts[i] = 0;
for (uint64_t i = 0; i < num_local_edges[t]*2; i+=2)
++temp_counts[edge_list[t][i]];
for (uint64_t i = 0; i < num_local_verts; ++i)
degree_list[t][i+1] = degree_list[t][i] + temp_counts[i];
memcpy(temp_counts, degree_list[t], num_local_verts*sizeof(uint64_t));
for (uint64_t i = 0; i < num_local_edges[t]*2; i+=2)
tmp_edges[temp_counts[edge_list[t][i]]++] = edge_list[t][i+1];
free(edge_list[t]);
edge_list[t] = tmp_edges;
if (createGhost) {
cur_label = num_local_verts;
std::vector<uint64_t> nonlocal_vids;
for (uint64_t i = 0; i < num_local_edges[t]; ++i) {
uint64_t out = edge_list[t][i];
if (vtx_mapping.find(out) ==vtx_mapping.end()) {
vtx_mapping[out] = cur_label;
edge_list[t][i] = cur_label++;
nonlocal_vids.push_back(out);
}
else
edge_list[t][i] = vtx_mapping[out];
}
num_ghost_verts = cur_label - num_local_verts;
if (num_ghost_verts>0) {
ghost_unmap = (uint64_t*)malloc(num_ghost_verts*sizeof(uint64_t));
owners = (int32_t*)malloc(num_ghost_verts*sizeof(int32_t));
for (uint64_t i = 0; i < nonlocal_vids.size(); ++i) {
ghost_unmap[i] = nonlocal_vids[i];
owners[i] = ranks[nonlocal_vids[i]];
}
}
}
return 0;
}
int main(int argc, char* argv[]) {
MPI_Init(&argc,&argv);
PCU_Comm_Init();
PCU_Debug_Open();
if ( argc != 2&&argc!=3 ) {
if ( !PCU_Comm_Self() )
printf("Usage: %s <binary_graph_file> [vertex_partition_file]",argv[0]);
PCU_Comm_Free();
MPI_Finalize();
assert(false);
}
agi::binGraph* g;
zagi::ZoltanCutVertex* ptn;
int self = PCU_Comm_Self();
int num_parts = PCU_Comm_Peers();
PCU_Switch_Comm(MPI_COMM_SELF);
if (!self) {
g = new agi::binGraph(argv[1]);
ptn = new zagi::ZoltanCutVertex(g,num_parts);
ptn->run();
}
else
g = new agi::binGraph();
agi::EdgePartitionMap map;
if (!self) {
ptn->createPtn(map);
delete ptn;
}
PCU_Switch_Comm(MPI_COMM_WORLD);
g->migrate(map);
partitionInfo(g);
delete g;
if (!PCU_Comm_Self())
printf("Block Partitioning\n");
g = new agi::binGraph(argv[1]);
partitionInfo(g);
delete g;
if (argc==3) {
if (!PCU_Comm_Self())
printf("Partitioned: %s\n",argv[2]);
g = new agi::binGraph(argv[1],argv[2]);
partitionInfo(g);
delete g;
}
PCU_Barrier();
if (!PCU_Comm_Self())
printf("\nAll tests passed\n");
PCU_Comm_Free();
MPI_Finalize();
}
static void note_remote_link(mds_id i, struct mds_copy c, void* u)
{
(void)i;
if (c.p != PCU_Comm_Self())
note_peer(u, c.p);
}