Exemplo n.º 1
0
void partitionInfo(agi::binGraph* g) {
  lid_t total_verts = g->numTotalVtxs();
  lid_t global_verts = g->numGlobalVtxs();
  lid_t edges = g->numLocalEdges()+g->numLocalEdges(SPLIT_TYPE);
  lid_t min = PCU_Min_Int(total_verts);
  lid_t max = PCU_Max_Int(total_verts);
  lid_t tot = PCU_Add_Long(total_verts);
  double avg = ((double)tot)/PCU_Comm_Peers();
  double imb = max/avg;
  double inc = ((double)(tot-global_verts))/global_verts*100;
  if (!PCU_Comm_Self()) 
    printf("Vertices: Min %lu Max %lu Tot %lu Inc %1.4f Avg %1.4f Imb %1.3f\n",min,max,tot,inc,avg,imb);
  min = PCU_Min_Int(edges);
  max = PCU_Max_Int(edges);
  tot = PCU_Add_Long(edges);
  avg = ((double)tot)/PCU_Comm_Peers();
  imb = max/avg;
  if (!PCU_Comm_Self()) 
    printf("Edges: Min %lu Max %lu Tot %lu Avg %1.4f Imb %1.3f\n",min,max,tot,avg,imb);
  
  lid_t edge_cut =0;
  agi::EdgeIterator* eitr = g->begin(0);
  for (int i=0;i<g->numLocalEdges();i++) {
    agi::GraphEdge* e = g->iterate(eitr);
    if (g->owner(g->v(e))!=PCU_Comm_Self())
      edge_cut++;
  }
  edge_cut+=g->numLocalEdges(SPLIT_TYPE);
  edge_cut = PCU_Add_Long(edge_cut);
  if (!PCU_Comm_Self())
    printf("Edge Cut: %lu\n",edge_cut);
}
Exemplo n.º 2
0
static void note_local_link(mds_id i, struct mds_copy c, void* u)
{
  if (c.p == PCU_Comm_Self()) {
    if (i < mds_index(c.e))
      note_peer(u, PCU_Comm_Self());
    else /* hack id to store self-receivers */
      note_peer(u, PCU_Comm_Peers());
  }
}
Exemplo n.º 3
0
static void take_local_link(mds_id i, struct mds_copy c, void* u)
{
  struct mds_links* ln = u;
  int self = find_peer(ln, PCU_Comm_Self());
  int other = find_peer(ln, PCU_Comm_Peers());
  mds_id j = mds_index(c.e);
  if ((PCU_Comm_Self() == c.p) && (i < j)) {
    ln->l[self][ln->n[self]] = i;
    ln->l[other][ln->n[other]] = j;
    /* use ns as (redundant) position keepers */
    ++ln->n[self];
    ++ln->n[other];
  }
}
Exemplo n.º 4
0
void mds_set_type_links(struct mds_net* net, struct mds* m,
    int t, struct mds_links* ln)
{
  unsigned i;
  unsigned j;
  unsigned* in;
  struct mds_copy c;
  PCU_Comm_Begin();
  for (i = 0; i < ln->np; ++i) {
    PCU_ALWAYS_ASSERT(ln->l);
    for (j = 0; j < ln->n[i]; ++j)
      PCU_COMM_PACK(ln->p[i], ln->l[i][j]);
  }
  PCU_Comm_Send();
  while (PCU_Comm_Listen()) {
    c.p = PCU_Comm_Sender();
    PCU_ALWAYS_ASSERT(c.p != PCU_Comm_Self());
    i = find_peer(ln, c.p);
    in = PCU_Comm_Extract(ln->n[i] * sizeof(unsigned));
    for (j = 0; j < ln->n[i]; ++j) {
      c.e = mds_identify(t, in[j]);
      mds_add_copy(net, m, mds_identify(t, ln->l[i][j]), c);
    }
  }
}
Exemplo n.º 5
0
static void send_remote_link(mds_id i, struct mds_copy c, void* u)
{
  (void)i;
  (void)u;
  if (PCU_Comm_Self() < c.p)
    PCU_COMM_PACK(c.p, c.e);
}
static void switchToOriginals(int npartitions)
{
  int self = PCU_Comm_Self();
  int groupRank = self / npartitions;
  int group = self % npartitions;
  MPI_Comm groupComm;
  MPI_Comm_split(MPI_COMM_WORLD, group, groupRank, &groupComm);
  PCU_Switch_Comm(groupComm);
}
Exemplo n.º 7
0
static void take_remote_link(mds_id i, struct mds_copy c, void* u)
{
  struct mds_links* ln = u;
  int pi;
  if (PCU_Comm_Self() < c.p) {
    pi = find_peer(ln, c.p);
    ln->l[pi][ln->n[pi]] = i;
    /* use n as position keeper */
    ++ln->n[pi];
  }
}
Exemplo n.º 8
0
static int owns_copies(mds_id e, struct mds_copies* c)
{
  int i;
  struct mds_copy mc;
  mc.e = e;
  mc.p = PCU_Comm_Self();
  for (i = 0; i < c->n; ++i)
    if (copy_less(c->c[i], mc))
      return 0;
  return 1;
}
Exemplo n.º 9
0
void mds_free_local_links(struct mds_links* ln)
{
  int self, other;
  self = find_peer(ln, PCU_Comm_Self());
  if (self == -1)
    return;
  other = find_peer(ln, PCU_Comm_Peers());
  ln->n[self] = ln->n[other] = 0;
  free(ln->l[self]);
  free(ln->l[other]);
  ln->l[self] = ln->l[other] = NULL;
}
Exemplo n.º 10
0
static int read_magic_number(FILE* f)
{
  int magic;
  if (!seek_after_header(f, magic_name)) {
    if (!PCU_Comm_Self())
      fprintf(stderr,"warning: not swapping bytes\n");
    rewind(f);
    return 0;
  }
  my_fread(&magic, sizeof(int), 1, f);
  return magic != MAGIC;
}
Exemplo n.º 11
0
  //TODO: optimize these operations using PCU
//Private Functions
etype binGraph::load_edges(char *filename, uint64_t*& read_edges,
    uint64_t& m_read) {  
  FILE *infp = fopen(filename, "rb");
  fseek(infp, 0L, SEEK_END);
  uint64_t file_size = ftell(infp);
  fseek(infp, 0L, SEEK_SET);
  etype t = addEdgeType();
  num_global_edges[t] = file_size/(2*sizeof(uint32_t));
  
  uint64_t read_offset_start = PCU_Comm_Self()*2*sizeof(uint32_t)*
    (num_global_edges[t] / (uint64_t)PCU_Comm_Peers());
  uint64_t read_offset_end = (PCU_Comm_Self()+1)*2*sizeof(uint32_t)*
    (num_global_edges[t] / (uint64_t)PCU_Comm_Peers());

  if (PCU_Comm_Self() == PCU_Comm_Peers() - 1)
    read_offset_end = 2*sizeof(uint32_t)*num_global_edges[t];

  m_read = (read_offset_end - read_offset_start)/(2*sizeof(uint32_t));

  uint32_t* temp_read = (uint32_t*)malloc(2*m_read*sizeof(uint32_t));
  read_edges = (uint64_t*)malloc(2*m_read*sizeof(uint64_t));
  fseek(infp, read_offset_start, SEEK_SET);
  fread(temp_read, m_read, 2*sizeof(uint32_t), infp);
  fclose(infp);
  for (uint64_t i = 0; i < m_read*2; ++i)
    read_edges[i] = (uint64_t)temp_read[i];
  free(temp_read);

  num_global_verts = 0;
  for (uint64_t i = 0; i < m_read*2; ++i)
    if (read_edges[i] > num_global_verts) {
      num_global_verts = read_edges[i];
    }

  MPI_Allreduce(MPI_IN_PLACE, &num_global_verts, 1, MPI_UINT64_T, MPI_MAX, PCU_Get_Comm());
 
  num_global_verts += 1;

  return t;
}
Exemplo n.º 12
0
void mds_set_local_matches(struct mds_net* net, struct mds* m,
                         int t, struct mds_links* ln)
{
  int self, other;
  unsigned i;
  mds_id a, b;
  struct mds_copy c;
  c.p = PCU_Comm_Self();
  self = find_peer(ln, PCU_Comm_Self());
  if (self == -1)
    return;
  other = find_peer(ln, PCU_Comm_Peers());
  assert(ln->n[self] = ln->n[other]);
  for (i = 0; i < ln->n[self]; ++i) {
    a = mds_identify(t, ln->l[self][i]);
    b = mds_identify(t, ln->l[other][i]);
    c.e = b;
    mds_add_copy(net, m, a, c);
    c.e = a;
    mds_add_copy(net, m, b, c);
  }
}
Exemplo n.º 13
0
void mds_get_local_matches(struct mds_net* net, struct mds* m,
                         int t, struct mds_links* ln)
{
  int self, other;
  for_type_net(net, m, t, note_local_link, ln);
  self = find_peer(ln, PCU_Comm_Self());
  if (self == -1)
    return;
  other = find_peer(ln, PCU_Comm_Peers());
  assert(ln->n[self] == ln->n[other]);
  ln->l[self] = malloc(ln->n[self] * sizeof(unsigned));
  ln->l[other] = malloc(ln->n[other] * sizeof(unsigned));
  ln->n[self] = 0;
  ln->n[other] = 0;
  for_type_net(net, m, t, take_local_link, ln);
}
Exemplo n.º 14
0
static int find_header(FILE* f, const char* name, char header[PH_LINE])
{
  char* hname;
  long bytes;
  char tmp[PH_LINE];
  while (fgets(header, PH_LINE, f)) {
    if ((header[0] == '#') || (header[0] == '\n'))
      continue;
    strncpy(tmp, header, PH_LINE-1);
    tmp[PH_LINE-1] = '\0';
    parse_header(tmp, &hname, &bytes, 0, NULL);
    if (!strncmp(name, hname, strlen(name)))
      return 1;
    fseek(f, bytes, SEEK_CUR);
  }
  if (!PCU_Comm_Self())
    fprintf(stderr,"warning: phIO could not find \"%s\"\n",name);
  return 0;
}
Exemplo n.º 15
0
/* algorithm courtesy of Sebastian Rettenberger:
   use brokers/routers for the vertex global ids.
   Although we have used this trick before (see mpas/apfMPAS.cc),
   I didn't think to use it here, so credit is given. */
static void constructResidence(Mesh2* m, GlobalToVert& globalToVert)
{
  Gid max = getMax(globalToVert);
  Gid total = max + 1;
  int peers = PCU_Comm_Peers();
  int quotient = total / peers;
  int remainder = total % peers;
  int mySize = quotient;
  int self = PCU_Comm_Self();
  if (self == (peers - 1))
    mySize += remainder;
  typedef std::vector< std::vector<int> > TmpParts;
  TmpParts tmpParts(mySize);
  /* if we have a vertex, send its global id to the
     broker for that global id */
  PCU_Comm_Begin();
  APF_ITERATE(GlobalToVert, globalToVert, it) {
    int gid = it->first;
    int to = std::min(peers - 1, gid / quotient);
    PCU_COMM_PACK(to, gid);
  }
Exemplo n.º 16
0
void mds_get_type_links(struct mds_net* net, struct mds* m,
    int t, struct mds_links* ln)
{
  unsigned i;
  /* count remote links in np, p and n */
  for_type_net(net, m, t, note_remote_link, ln);
  alloc_links(ln);
  for (i = 0; i < ln->np; ++i)
    if (((unsigned)PCU_Comm_Self()) < ln->p[i])
      /* zero n's for behavior in take_copy */
      ln->n[i] = 0;
  /* record indices in local order for owned boundaries */
  for_type_net(net, m, t, take_remote_link, ln);
  PCU_Comm_Begin();
  /* send indices in local order for owned boundaries */
  for_type_net(net, m, t, send_remote_link, NULL);
  PCU_Comm_Send();
  while (PCU_Comm_Listen())
  /* recv indices in remote order for non-owned boundaries */
    recv_links(ln);
}
Exemplo n.º 17
0
Albany::PUMIMeshStruct::PUMIMeshStruct(
    const Teuchos::RCP<Teuchos::ParameterList>& params,
    const Teuchos::RCP<const Teuchos_Comm>& commT)
{
  params->validateParameters(
      *(PUMIMeshStruct::getValidDiscretizationParameters()), 0);

  outputFileName = params->get<std::string>("PUMI Output File Name", "");
  outputInterval = params->get<int>("PUMI Write Interval", 1); // write every time step default

  std::string model_file;
  if(params->isParameter("Mesh Model Input File Name"))
    model_file = params->get<std::string>("Mesh Model Input File Name");

#ifdef SCOREC_SIMMODEL
  if (params->isParameter("Acis Model Input File Name"))
    model_file = params->get<std::string>("Parasolid Model Input File Name");

  if(params->isParameter("Parasolid Model Input File Name"))
    model_file = params->get<std::string>("Parasolid Model Input File Name");
#endif

  if (params->isParameter("PUMI Input File Name")) {

    std::string mesh_file = params->get<std::string>("PUMI Input File Name");
    mesh = 0;

    // If we are running in parallel but have a single mesh file, split it and rebalance
    bool useSerialMesh = params->get<bool>("Use Serial Mesh", false);
    if (useSerialMesh && commT->getSize() > 1){ // do the equivalent of the SCOREC "split" utility
       apf::Migration* plan = 0;
       gmi_model* g = 0;
       g = gmi_load(model_file.c_str());
       bool isOriginal = ((PCU_Comm_Self() % commT->getSize()) == 0);
       switchToOriginals(commT->getSize());
       if (isOriginal) {
         mesh = apf::loadMdsMesh(g, mesh_file.c_str());
         plan = getPlan(mesh, commT->getSize());
       }
       switchToAll();
       mesh = repeatMdsMesh(mesh, g, plan, commT->getSize());
    }
    else {
      mesh = apf::loadMdsMesh(model_file.c_str(), mesh_file.c_str());
    }

  } else {
    int nex = params->get<int>("1D Elements", 0);
    int ney = params->get<int>("2D Elements", 0);
    int nez = params->get<int>("3D Elements", 0);
    double wx = params->get<double>("1D Scale", 1);
    double wy = params->get<double>("2D Scale", 1);
    double wz = params->get<double>("3D Scale", 1);
    bool is = ! params->get<bool>("Hexahedral", true);
    buildBoxMesh(nex, ney, nez, wx, wy, wz, is);
  }

  model = mesh->getModel();

  // Tell the mesh that we'll handle deleting the model.
  apf::disownMdsModel(mesh);

  bool isQuadMesh = params->get<bool>("2nd Order Mesh",false);
  if (isQuadMesh)
    apf::changeMeshShape(mesh, apf::getSerendipity(), false);

  // Resize mesh after input if indicated in the input file
  // User has indicated a desired element size in input file
  if(params->isParameter("Resize Input Mesh Element Size")){
    SizeFunction sizeFunction(params->get<double>(
          "Resize Input Mesh Element Size", 0.1));
    int num_iters = params->get<int>(
        "Max Number of Mesh Adapt Iterations", 1);
    ma::Input* input = ma::configure(mesh,&sizeFunction);
    input->maximumIterations = num_iters;
    input->shouldSnap = false;
    ma::adapt(input);
  }

  // get the continuation step to write a restart file
  restartWriteStep = params->get<int>("Write Restart File at Step",0);

  APFMeshStruct::init(params, commT);

  // if we have a restart time, we will want to override some of
  // the default paramaters set by APFMeshStruct::init
  if (params->isParameter("PUMI Restart Time")) {
    hasRestartSolution = true;
    restartDataTime = params->get<double>("PUMI Restart Time", 0.0);
    std::string name = params->get<std::string>("PUMI Input File Name");
    if (!PCU_Comm_Self())
      std::cout << "Restarting from time: " << restartDataTime
        << " from restart file: " << name << std::endl;
  }

  if (params->isParameter("Load FELIX Data"))
    shouldLoadFELIXData = true;

}
Exemplo n.º 18
0
void binGraph::migrate(agi::EdgePartitionMap& map) {
  EdgePartitionMap::iterator itr;
  PCU_Comm_Begin();
  for (itr = map.begin();itr!=map.end();itr++) {
    lid_t lid = itr->first;
    gid_t v1 = local_unmap[u(lid)];
    gid_t v2 = local_unmap[edge_list[0][lid]];
    PCU_COMM_PACK(itr->second,v1);
    PCU_COMM_PACK(itr->second,v2);
  }
  PCU_Comm_Send();
  std::vector<gid_t> recv_edges;
  while (PCU_Comm_Receive()) {
    gid_t v1;
    PCU_COMM_UNPACK(v1);
    recv_edges.push_back(v1);
  }
  num_global_verts = PCU_Add_Long(num_global_verts);
  if (numEdgeTypes()==0)
    addEdgeType();

  num_local_edges[0] = recv_edges.size()/2;
  num_global_edges[0] = PCU_Add_Long(num_local_edges[0]);
  
  if (edge_list[0])
    delete edge_list[0];
  edge_list[0] = new gid_t[num_local_edges[0]*2];
  std::copy(recv_edges.begin(),recv_edges.end(),edge_list[0]);
  vtx_mapping.clear();
  int32_t* ranks = (int32_t*)malloc(num_global_verts*sizeof(int32_t));
  for (int i=0;i<num_global_verts;i++)
    ranks[i] = -1;
  for (lid_t i=0;i<num_local_edges[0]*2;i++) {
    ranks[edge_list[0][i]] = PCU_Comm_Self();
  }

  create_dist_csr(ranks,0,false);
  delete [] ranks;

  //TODO: Make much more efficient
  PCU_Comm_Begin();
  for (int i=0;i<num_local_verts;i++) {
    for (int j=1;j<PCU_Comm_Peers();j++)
      PCU_COMM_PACK((PCU_Comm_Self()+j)%PCU_Comm_Peers(),local_unmap[i]);
  }
  PCU_Comm_Send();
  std::vector<part_t> owns;
  std::vector<gid_t> dups;
  degree_list[SPLIT_TYPE] = new lid_t[num_local_verts+1];
  for (int i=0;i<num_local_verts+1;++i)
    degree_list[SPLIT_TYPE][i]=0;

  while (PCU_Comm_Receive()) {
    gid_t gid;
    PCU_COMM_UNPACK(gid);
    map_t::iterator itr = vtx_mapping.find(gid);
    if (itr!=vtx_mapping.end()) {
      dups.push_back(gid);
      owns.push_back(PCU_Comm_Sender());
      degree_list[SPLIT_TYPE][itr->second+1]++;
    }
  }

  for (int i=1;i<num_local_verts+1;++i)
    degree_list[SPLIT_TYPE][i]+=degree_list[SPLIT_TYPE][i-1];

  assert(degree_list[SPLIT_TYPE][num_local_verts] ==dups.size());
  num_ghost_verts = dups.size();
  num_local_edges[SPLIT_TYPE] = dups.size();

  ghost_unmap = new gid_t[dups.size()];
  owners = new part_t[dups.size()];
  uint64_t* temp_counts = (uint64_t*)malloc(num_local_verts*sizeof(uint64_t));
  memcpy(temp_counts, degree_list[SPLIT_TYPE], num_local_verts*sizeof(uint64_t));

  edge_list[SPLIT_TYPE] = new lid_t[dups.size()];
  
  for (unsigned int i=0;i<dups.size();i++) {
    lid_t lid = vtx_mapping[dups[i]];
    edge_list[SPLIT_TYPE][temp_counts[lid]++] = num_local_verts+i;
    ghost_unmap[i]=dups[i];
    owners[i] = owns[i];
  }
  num_global_edges[SPLIT_TYPE] = PCU_Add_Long(num_local_edges[SPLIT_TYPE]);
}
Exemplo n.º 19
0
 void phi() {
   if (!PCU_Comm_Self()) {
     printf("hi\n");
   }
 }
Exemplo n.º 20
0
  int binGraph::create_dist_csr(int32_t* ranks,etype t,bool createGhost)
{
  num_local_verts = 0;

  for (uint64_t i = 0; i < num_global_verts; ++i)
    if (ranks[i] == PCU_Comm_Self())
      ++num_local_verts;

  local_unmap = (uint64_t*)malloc(num_local_verts*sizeof(uint64_t));
  uint64_t cur_label = 0;
  for (uint64_t i = 0; i < num_local_edges[t]*2; i++) {
    uint64_t out = edge_list[t][i];


    
    if (ranks[out] != PCU_Comm_Self())
      continue;
    if (vtx_mapping.count(out) == 0) {
      vtx_mapping[out] = cur_label;
      local_unmap[cur_label] = out;
      edge_list[t][i] = cur_label++;
    }
    else        
      edge_list[t][i] = vtx_mapping[out];
  }
  uint64_t* tmp_edges = (uint64_t*)malloc(num_local_edges[t]*sizeof(uint64_t));
  uint64_t* temp_counts = (uint64_t*)malloc(num_local_verts*sizeof(uint64_t));
  degree_list[t] = (uint64_t*)malloc((num_local_verts+1)*sizeof(uint64_t));
  for (uint64_t i = 0; i < num_local_verts+1; ++i)
    degree_list[t][i] = 0;
  for (uint64_t i = 0; i < num_local_verts; ++i)
    temp_counts[i] = 0;
  for (uint64_t i = 0; i < num_local_edges[t]*2; i+=2)
    ++temp_counts[edge_list[t][i]];
  for (uint64_t i = 0; i < num_local_verts; ++i)
    degree_list[t][i+1] = degree_list[t][i] + temp_counts[i];
  memcpy(temp_counts, degree_list[t], num_local_verts*sizeof(uint64_t));
  for (uint64_t i = 0; i < num_local_edges[t]*2; i+=2)
    tmp_edges[temp_counts[edge_list[t][i]]++] = edge_list[t][i+1];
  free(edge_list[t]);
  edge_list[t] = tmp_edges;
  if (createGhost) {
    cur_label = num_local_verts;
    std::vector<uint64_t> nonlocal_vids;
    for (uint64_t i = 0; i < num_local_edges[t]; ++i) {
      uint64_t out = edge_list[t][i];
      if (vtx_mapping.find(out) ==vtx_mapping.end()) {
	vtx_mapping[out] = cur_label;
	edge_list[t][i] = cur_label++;
	nonlocal_vids.push_back(out);
      }
      else        
	edge_list[t][i] = vtx_mapping[out];
    }
    num_ghost_verts = cur_label - num_local_verts;
    if (num_ghost_verts>0) {
      ghost_unmap = (uint64_t*)malloc(num_ghost_verts*sizeof(uint64_t));
      owners = (int32_t*)malloc(num_ghost_verts*sizeof(int32_t));
      for (uint64_t i = 0; i < nonlocal_vids.size(); ++i) {
	ghost_unmap[i] = nonlocal_vids[i];
	owners[i] = ranks[nonlocal_vids[i]];
      }
    }
  }

  return 0;
}
Exemplo n.º 21
0
int main(int argc, char* argv[]) {
  MPI_Init(&argc,&argv);
  PCU_Comm_Init();
  PCU_Debug_Open();
  if ( argc != 2&&argc!=3 ) {
    if ( !PCU_Comm_Self() )
      printf("Usage: %s <binary_graph_file> [vertex_partition_file]",argv[0]);
    PCU_Comm_Free();
    MPI_Finalize();
    assert(false);
  }
  agi::binGraph* g;
  zagi::ZoltanCutVertex* ptn;
  int self = PCU_Comm_Self();
  int num_parts = PCU_Comm_Peers();
  PCU_Switch_Comm(MPI_COMM_SELF);
  if (!self) {

    g = new agi::binGraph(argv[1]);
    ptn = new zagi::ZoltanCutVertex(g,num_parts);
    ptn->run();
  }
  else
    g = new agi::binGraph();
  agi::EdgePartitionMap map;
  if (!self) {
    ptn->createPtn(map);
    delete ptn;
  }

  PCU_Switch_Comm(MPI_COMM_WORLD);
  g->migrate(map);

  partitionInfo(g);

  delete g;

  if (!PCU_Comm_Self())
    printf("Block Partitioning\n");

  g = new agi::binGraph(argv[1]);

  partitionInfo(g);

  delete g;

  if (argc==3) {
  if (!PCU_Comm_Self())
    printf("Partitioned: %s\n",argv[2]);
    g = new agi::binGraph(argv[1],argv[2]);
    partitionInfo(g);
    delete g;
  }

  PCU_Barrier();
  if (!PCU_Comm_Self())
    printf("\nAll tests passed\n");

  PCU_Comm_Free();
  MPI_Finalize();

}
Exemplo n.º 22
0
static void note_remote_link(mds_id i, struct mds_copy c, void* u)
{
  (void)i;
  if (c.p != PCU_Comm_Self())
    note_peer(u, c.p);
}