void IWeights::ChangeCoords(const IWeights &src_uv, const IWeights &dst_uv) {
Trace __trace("IWeights::ChangeCoords(const IWeights &src_uv, const IWeights &dst_uv)");
struct UV_vect {
double U[3];
double V[3];
};
WeightMap new_weights; // will create whole new matrix
// Loop rows in matrix
WeightMap::iterator wi = weights.begin(), we = weights.end();
for (; wi != we;) {
// There must be 3 consecutive entries with this id (index 0,1,2)
std::vector<const Entry*> rows(3);
std::vector<std::vector<Entry>*> cols(3);
// long id = wi->first.id;
// Get three rows. Since weights are redundant, reuse
{
rows[0] = &wi->first; cols[0] = &wi->second;
ThrowRequire(wi->first.idx == 0);
rows[1] = &wi->first; cols[1] = &wi->second;
rows[2] = &wi->first; cols[2] = &wi->second;
++wi;
}
// Lookup UV destination for this row. These are stored as
// (U V)^T, i.e.
// (gid, 0) -> (gid, 0, Ux) (gid, 1, Uy) (gid, 2, Uz)
// (gid, 1) -> (gid, 0, Vx) (gid, 1, Vy) (gid, 2, Vz)
WeightMap::const_iterator ruv_i = dst_uv.weights.lower_bound(Entry(rows[0]->id, 0));
UV_vect ruv;
for (UInt r = 0; r < 2; r++) {
ThrowRequire(ruv_i != dst_uv.weights.end());
ThrowRequire(ruv_i->first.id == rows[0]->id);
ThrowRequire((UInt) ruv_i->first.idx == r);
{
const std::vector <Entry> &ruv_col = ruv_i->second;
ThrowRequire(ruv_col.size() == 3);
double *val = r == 0 ? ruv.U : ruv.V;
val[0] = ruv_col[0].value;
val[1] = ruv_col[1].value;
val[2] = ruv_col[2].value;
}
++ruv_i;
}
// Loop columns
UInt ncols = cols[0]->size();
ThrowRequire(ncols == cols[1]->size() && ncols == cols[2]->size());
Entry row_u(*rows[0]);
Entry row_v(*rows[1]); row_v.idx = 1;
std::vector<std::vector<Entry> > new_cols(2, std::vector<Entry>(2*ncols));
for (UInt i = 0; i < ncols; i++) {
// Get the UV for row
Entry &col_x = cols[0]->operator[](i);
Entry &col_y = cols[1]->operator[](i);
Entry &col_z = cols[2]->operator[](i);
// Entries should reference the same id
UInt col_id = col_x.id;
ThrowRequire(col_id == col_y.id && col_id == col_z.id);
// Get column UV vectors (source)
// stored as (U V), i.e.
// (gid, 0) -> (gid, 0, Ux) (gid, 1, Vx)
// (gid, 1) -> (gid, 0, Uy) (gid, 1, Vy)
// (gid, 2) -> (gid, 0, Uz) (gid, 1, Vz)
// Get the column uv vectors
WeightMap::const_iterator cuv_i = src_uv.weights.lower_bound(Entry(col_id, 0));
UV_vect cuv;
for (UInt r = 0; r < 3; r++) {
ThrowRequire(cuv_i != src_uv.weights.end() && (UInt) cuv_i->first.id == col_id && (UInt) cuv_i->first.idx == r);
{
const std::vector <Entry> &cuv_col = cuv_i->second;
ThrowRequire(cuv_col.size() == 2);
cuv.U[r] = cuv_col[0].value;
cuv.V[r] = cuv_col[1].value;
}
++cuv_i;
}
// Add the u entries
{
Entry &u_col_u = new_cols[0][2*i];
Entry &u_col_v = new_cols[0][2*i+1];
u_col_u.id = u_col_v.id = col_x.id;
u_col_u.idx = 0; u_col_v.idx = 1;
// After all this data structure ado, the heart of the
// numerical algorithm:
u_col_u.value = ruv.U[0]*col_x.value*cuv.U[0] +
ruv.U[1]*col_y.value*cuv.U[1] +
ruv.U[2]*col_z.value*cuv.U[2];
u_col_v.value = ruv.U[0]*col_x.value*cuv.V[0] +
ruv.U[1]*col_y.value*cuv.V[1] +
ruv.U[2]*col_z.value*cuv.V[2];
}
// Add the v entries
{
Entry &v_col_u = new_cols[1][2*i];
Entry &v_col_v = new_cols[1][2*i+1];
v_col_u.id = v_col_v.id = col_x.id;
v_col_u.idx = 0; v_col_v.idx = 1;
v_col_u.value = ruv.V[0]*col_x.value*cuv.U[0] +
ruv.V[1]*col_y.value*cuv.U[1] +
ruv.V[2]*col_z.value*cuv.U[2];
v_col_v.value = ruv.V[0]*col_x.value*cuv.V[0] +
ruv.V[1]*col_y.value*cuv.V[1] +
ruv.V[2]*col_z.value*cuv.V[2];
}
} // for i
// Insert new rows
new_weights[row_u] = new_cols[0];
new_weights[row_v] = new_cols[1];
} // for wi
weights.swap(new_weights);
}
// *** Convert a grid to a mesh. The staggerLoc should describe
// whether the mesh is at the dual location or coincident with the
// grid itself. I am not sure how this is going to work, though,
// since the dual of a nice multi-patch grid could be a very bad
// object, not representable by a mesh.
//
// o------------o-------------o
// | : | : |
// | : | : |
// |.....x......|......x......|
// | : | : |
// | : | : |
// o------------o-------------o
// | : | : |
// | : | : |
// |.....x......|......x......|
// | : | : |
// | : | : |
// o------------o-------------o
//
// E.G. mesh o---o, dual mesh x....x
//
// I think there is work here.
// For the moment, we should at least be able to represent the grid itself,
// and, maybe, for simple single patch grids with a periodic component,
// the dual, which is not so bad. This will put us equivalent with
// SCRIP.
void GridToMesh(const Grid &grid_, int staggerLoc, ESMCI::Mesh &mesh, const std::vector<ESMCI::Array*> &arrays) {
Trace __trace("GridToMesh(const Grid &grid_, int staggerLoc, ESMCI::Mesh &mesh)");
// Initialize the parallel environment for mesh (if not already done)
ESMCI::Par::Init("MESHLOG", false /* use log */);
Grid &grid = const_cast<Grid&>(grid_);
bool is_sphere = grid.isSphere();
try {
// *** Grid error checking here ***
// *** Set some meta-data ***
// We set the topological dimension of the mesh (quad = 2, hex = 3, etc...)
UInt pdim = grid.getDimCount();
mesh.set_parametric_dimension(pdim);
// In what dimension is the grid embedded?? (sphere = 3, simple rectangle = 2, etc...)
// At this point the topological and spatial dim of the Grid is the same this should change soon
UInt sdim = grid.getDimCount();
if (is_sphere) {
//std::cout << "g2m, is sphere=1" << std::endl;
sdim = 3;
}
mesh.set_spatial_dimension(sdim);
// We save the nodes in a linear list so that we can access then as such
// for cell creation.
//TODO: NEED MAX LID METHOD SOON ->Bob
std::vector<MeshObj*> nodevect(100000);
std::map<UInt,UInt> ngid2lid;
UInt local_node_num = 0, local_elem_num = 0;
// Set the id of this processor here (me)
int localrc;
int rc;
int me = VM::getCurrent(&localrc)->getLocalPet();
// if (ESMC_LogDefault.MsgFoundError(localrc, ESMF_ERR_PASSTHRU, &rc)) // How should I handle ESMF LogError?
// return;
// Keep track of locally owned, shared and shared, not-locally-owned
std::vector<UInt> owned_shared;
std::vector<UInt> notowned_shared;
// *** Create the Nodes ***
// Loop nodes of the grid. Here we loop all nodes, both owned and not.
ESMCI::GridIter *gni=new ESMCI::GridIter(&grid,staggerLoc,true);
// Put Local in first, so we don't have to search for duplicates for every interation,
// after locals are in put in non-local
// loop through all LOCAL nodes in the Grid owned by cells
for(gni->toBeg(); !gni->isDone(); gni->adv()) {
// Only operate on Local Nodes
if (gni->isLocal()) {
MeshObj *node;
// get the global id of this Grid node
int gid=gni->getGlobalID();
// get the local id of this Grid node
int lid=gni->getLocalID();
#ifdef G2M_DBG
Par::Out() << "GID=" << gid << ", LID=" << lid << std::endl;
#endif
// Create new node in mesh object
node = new MeshObj(MeshObj::NODE, // node...
gid, // unique global id
local_node_num
);
ngid2lid[gid] = lid;
local_node_num++;
node->set_owner(me); // Set owner to this proc
UInt nodeset = is_sphere ? gni->getPoleID() : 0; // Do we need to partition the nodes in any sets?
mesh.add_node(node, nodeset);
// If Shared add to list to use DistDir on
if (gni->isShared()) {
// Put gid in list
std::vector<UInt>::iterator lb =
std::lower_bound(owned_shared.begin(), owned_shared.end(), gid);
if (lb == owned_shared.end() || *lb != gid)
owned_shared.insert(lb, gid);
}
// Put node into vector
nodevect[lid]=node;
}
} // gni
// loop through all NON-LOCAL nodes in the Grid owned by cells
for(gni->toBeg(); !gni->isDone(); gni->adv()) {
// Only operate on non-local nodes
if (!gni->isLocal()) {
MeshObj *node;
// get the global id of this Grid node
int gid=gni->getGlobalID();
// get the local id of this Grid node
int lid=gni->getLocalID();
// If Grid node is not already in the mesh then add
Mesh::MeshObjIDMap::iterator mi = mesh.map_find(MeshObj::NODE, gid);
if (mi == mesh.map_end(MeshObj::NODE)) {
node = new MeshObj(MeshObj::NODE, // node...
gid, // unique global id
local_node_num // local ID for boostrapping field data
);
ngid2lid[gid] = lid;
local_node_num++;
node->set_owner(std::numeric_limits<UInt>::max()); // Set owner to unknown (will have to ghost later)
UInt nodeset = is_sphere ? gni->getPoleID() : 0; // Do we need to partition the nodes in any sets?
mesh.add_node(node, nodeset);
// Node must be shared
std::vector<UInt>::iterator lb =
std::lower_bound(notowned_shared.begin(), notowned_shared.end(), gid);
if (lb == notowned_shared.end() || *lb != gid)
notowned_shared.insert(lb, gid);
} else {
node=&*mi;
}
// Put node into vector
nodevect[lid]=node;
}
} // gni
// Use DistDir to fill node owners for non-local nodes
// TODO: Use nodes which are gni->isLocal() && gni->isShared() to get owners of
// non-local nodes ->David
// *** Create the Cells ***
// Presumably, for a structured grid there is only the
// 'hypercube' topology, i.e. a quadrilateral for 2d
// and a hexahedron for 3d
const MeshObjTopo *ctopo = 0;
if (pdim == 2) {
ctopo = sdim == 2 ? GetTopo("QUAD") : GetTopo("QUAD_3D");
} else if (pdim == 3) {
ThrowRequire(sdim == 3);
ctopo = GetTopo("HEX");
} else Throw() << "Invalid parametric dim:" << pdim;
if (!ctopo)
Throw() << "Could not get topology for pdim=" << pdim;
// Allocate vector to hold nodes for translation
std::vector<MeshObj*> nodes(ctopo->num_nodes);
// Loop Cells of the grid.
ESMCI::GridCellIter *gci=new ESMCI::GridCellIter(&grid,staggerLoc);
for(gci->toBeg(); !gci->isDone(); gci->adv()) {
MeshObj *cell = new MeshObj(MeshObj::ELEMENT, // Mesh equivalent of Cell
gci->getGlobalID(), // unique global id
local_elem_num++
);
#ifdef G2M_DBG
Par::Out() << "Cell:" << cell->get_id() << " uses nodes:";
#endif
// Set Owner
cell->set_owner(me);
// Get Local Ids of Corners
int cnrCount;
int cnrList[16]; // ONLY WORKS FOR UP TO 4D
gci->getCornersCellNodeLocalID(&cnrCount, cnrList);
ThrowRequire(cnrCount == ctopo->num_nodes);
// Get Nodes via Local IDs
for (UInt n = 0; n < ctopo->num_nodes; ++n) {
nodes[n] = nodevect[cnrList[n]];
#ifdef G2M_DBG
Par::Out() << nodes[n]->get_id() << " ";
#endif
} // n
#ifdef G2M_DBG
Par::Out() << std::endl;
#endif
UInt block_id = 1; // Any reason to use different sets for cells?
mesh.add_element(cell, nodes, block_id, ctopo);
} // ci
// Remove any superfluous nodes
mesh.remove_unused_nodes();
mesh.linearize_data_index();
// Now set up the nodal coordinates
IOField<NodalField> *node_coord = mesh.RegisterNodalField(mesh, "coordinates", sdim);
// Create whatever fields the user wants
std::vector<IOField<NodalField>*> nfields;
for (UInt i = 0; i < arrays.size(); ++i) {
char buf[512];
std::sprintf(buf, "array_%03d", i);
nfields.push_back(
mesh.RegisterNodalField(mesh, buf, 1)
);
nfields.back()->set_output_status(true);
}
#ifdef G2M_DBG
IOField<NodalField> *de_field = mesh.RegisterNodalField(mesh, "de", 1);
de_field->set_output_status(true);
#endif
// Loop through Mesh nodes setting up coordinates
MeshDB::iterator ni = mesh.node_begin(), ne = mesh.node_end();
for (; ni != ne; ++ni) {
double *c = node_coord->data(*ni);
double fdata;
UInt lid = ngid2lid[ni->get_id()]; // we set this above when creating the node
//Par::Out() << "node:" << ni->get_id() << ", lid=" << lid;
// Move to corresponding grid node
gni->moveToLocalID(lid);
// If local fill in coords
if (gni->isLocal()) {
gni->getCoord(c);
double DEG2RAD = M_PI/180.0;
if (is_sphere) {
double lon = c[0];
double lat = c[1];
double ninety = 90.0;
double theta = DEG2RAD*lon, phi = DEG2RAD*(ninety-lat);
c[0] = std::cos(theta)*std::sin(phi);
c[1] = std::sin(theta)*std::sin(phi);
c[2] = std::cos(phi);
}
} else { // set to Null value to be ghosted later
for (int i=0; i<sdim; i++) {
// c[i]=std::numeric_limits<double>::max();
c[i]=-10;
}
}
// Other arrays
for (UInt i = 0; i < arrays.size(); ++i) {
gni->getArrayData(arrays[i], &fdata);
double *data = nfields[i]->data(*ni);
ThrowRequire(data);
data[0] = fdata;
}
#ifdef G2M_DBG
// De field
double *data = de_field->data(*ni);
ThrowRequire(data);
data[0] = gni->getDE();
#endif
//printf("%d :: %f %f\n",gni->getGlobalID(),c[0],c[1]);
} // ni
// Now go back and resolve the ownership for shared nodes. We create a distdir
// from the owned, shared nodes, then look up the shared, not owned.
{
DDir<> dir;
std::vector<UInt> lids(owned_shared.size(), 0);
dir.Create(owned_shared.size(), &owned_shared[0], &lids[0]);
std::vector<DDir<>::dentry> lookups;
dir.RemoteGID(notowned_shared.size(), ¬owned_shared[0], lookups);
// Loop through the results. Do a map lookup to find nodes--since the shared
// porition of the mesh is a hypersurface, this should be cheap enough to do.
std::vector<DDir<>::dentry>::iterator ri = lookups.begin(), re = lookups.end();
for (; ri != re; ++ri) {
DDir<>::dentry &dent = *ri;
#ifdef G2M_DBG
Par::Out() << "Finding owner for gid" << dent.gid << " = " << dent.origin_proc << std::endl;
#endif
Mesh::MeshObjIDMap::iterator mi = mesh.map_find(MeshObj::NODE, dent.gid);
// ThrowRequire(mi != mesh.map_end(MeshObj::NODE));
// May have been deleted as an unused node.
if (mi == mesh.map_end(MeshObj::NODE)) {
#ifdef G2M_DBG
Par::Out() << "\tnot in mesh!!" << std::endl;
#endif
continue;
}
mi->set_owner(dent.origin_proc);
} // ri
} // ddir lookup
// Can now build the communication pattern.
mesh.build_sym_comm_rel(MeshObj::NODE);
// ** That's it. The mesh is now in the pre-commit phase, so other
// fields can be registered, sides/faces can be turned on, etc, or one
// can simply call mesh.Commit() and then proceed.
char buf[512];
//std::sprintf(buf, "g2m.%05d.txt", )
#ifdef G2M_DBG
mesh.Print(Par::Out());
#endif
mesh.Commit();
{
std::vector<MEField<>*> fds;
Mesh::FieldReg::MEField_iterator fi = mesh.Field_begin(), fe = mesh.Field_end();
for (; fi != fe; ++fi) fds.push_back(&*fi);
mesh.HaloFields(fds.size(), &fds[0]);
/*
MEField<> *cptr = mesh.GetCoordField();
mesh.HaloFields(1, &cptr);
*/
}
} // try catch block
catch (std::exception &x) {
// Set any ESMF error codes/messages
} // catch
catch(...) {
// Set any ESMF error codes/messages
}
}
/*
* Patch interpolation, where destination and source fields are nodal.
*/
void patch_serial_transfer(MEField<> &src_coord_field, UInt _nfields, MEField<>* const* _sfields, _field* const *_dfields, const std::vector<Interp::FieldPair> &fpairs, SearchResult &sres, Mesh &srcmesh) {
Trace __trace("patch_serial_transfer(MEField<> &src_coord_field, UInt _nfields, MEField<>* const* _sfields, _field* const *_dfields, int *iflag, SearchResult &sres)");
std::set<int> pdeg_set;
std::map<int, ElemPatch<>*> patch_map;
if (_nfields == 0) return;
// Get mask field pointer
MEField<> *src_mask_ptr = srcmesh.GetField("mask");
std::vector<MEField<>* > fields;
std::vector<_field* > dfields;
std::vector<int> orders;
UInt nrhs = 0;
for (UInt i = 0; i < _nfields; i++) {
// Only process interp_patch
if (fpairs[i].idata != Interp::INTERP_PATCH) continue;
pdeg_set.insert(fpairs[i].patch_order);
//ThrowRequire(_sfields[i]->is_nodal());
fields.push_back(_sfields[i]); dfields.push_back(_dfields[i]);
orders.push_back(fpairs[i].patch_order);
nrhs += _sfields[i]->dim(); // how many rhs for recovery
}
// Create the recovery field
SearchResult::iterator sb = sres.begin(), se = sres.end();
for (; sb != se; sb++) {
Search_result &sres = **sb;
// Trick: Gather the data from the source field so we may call interpolate point
const MeshObj &elem = *(*sb)->elem;
//std::cout << "Transfer: elem:" << elem.get_id() << std::endl;
{
std::set<int>::iterator pi = pdeg_set.begin(), pe = pdeg_set.end();
for (; pi != pe; ++pi) {
ElemPatch<> *epatch = new ElemPatch<>();
epatch->CreateElemPatch(*pi, ElemPatch<>::GAUSS_PATCH,
elem,
src_coord_field,
src_mask_ptr,
fields.size(),
&fields[0],
700000
);
patch_map[*pi] = epatch;
}
}
// Gather parametric coords into an array.
UInt pdim = GetMeshObjTopo(elem)->parametric_dim;
UInt npts = sres.nodes.size(); // number of points to interpolate
std::vector<double> pc(pdim*npts);
for (UInt np = 0; np < npts; np++) {
for (UInt pd = 0; pd < pdim; pd++) {
pc[np*pdim+pd] = sres.nodes[np].pcoord[pd];
}
}
std::map<int, std::vector<double> > result;
{
std::set<int>::iterator pi = pdeg_set.begin(), pe = pdeg_set.end();
for (; pi != pe; ++pi) {
std::pair<std::map<int, std::vector<double> >::iterator, bool> ri
= result.insert(std::make_pair(*pi, std::vector<double>()));
ThrowRequire(ri.second == true);
ri.first->second.resize(nrhs*npts);
ElemPatch<> &epatch = *patch_map[*pi];
epatch.Eval(npts, &pc[0], &(ri.first->second[0]));
}
}
std::vector<std::vector<double>* > field_results;
for (UInt i = 0; i < dfields.size(); i++)
field_results.push_back(&result[orders[i]]);
// Now copy data into fields
for (UInt np = 0; np < npts; np++) {
const MeshObj &snode = *sres.nodes[np].node;
UInt cur_field = 0;
for (UInt i = 0; i < dfields.size(); i++) {
const _field &dfield = *dfields[i];
std::vector<double> &results = *field_results[i];
if (dfield.OnObj(snode)) {
UInt fdim = dfield.dim(snode);
double *data = dfield.data(snode);
for (UInt f = 0; f < fdim; f++) {
data[f] = results[np*nrhs+(cur_field+f)];
}
}
cur_field += dfield.dim(snode);
} // for fi
} // for np
std::map<int, ElemPatch<>*>::iterator pi = patch_map.begin(), pe = patch_map.end();
for (; pi != pe; ++pi) delete pi->second;
} // for searchresult
}
void point_serial_transfer(Mesh &srcrend, Interp::UserFunc *ufunc, UInt num_fields, MEField<> *const *sfields, _field *const *dfields, int *iflag, SearchResult &sres) {
Trace __trace("point_serial_transfer(Interp::UserFunc *ufunc, UInt num_fields, MEField<> *const *sfields, _field *const *dfields, int *iflag, SearchResult &sres)");
if (num_fields == 0) return;
SearchResult::iterator sb = sres.begin(), se = sres.end();
for (; sb != se; sb++) {
Search_result &sres = **sb;
// Trick: Gather the data from the source field so we may call interpolate point
const MeshObj &elem = *(*sb)->elem;
UInt pdim = GetMeshObjTopo(elem)->parametric_dim;
// Inner loop through fields
for (UInt i = 0; i < num_fields; i++) {
if (iflag[i] != Interp::INTERP_STD) continue;
MEField<> &sfield = *sfields[i];
_field &dfield = *dfields[i];
// Load Parametric coords
UInt npts = sres.nodes.size(); // number of points to interpolate
std::vector<double> pcoord(pdim*npts);
for (UInt np = 0; np < npts; np++) {
for (UInt pd = 0; pd < pdim; pd++)
pcoord[np*pdim+pd] = sres.nodes[np].pcoord[pd];
}
std::vector<double> ires(npts*sfield.dim());
if (!ufunc) {
MEValues<> mev(sfield.GetMEFamily());
arbq pintg(pdim, npts, &pcoord[0]);
mev.Setup(elem, MEV::update_sf, &pintg);
mev.ReInit(elem);
mev.GetFunctionValues(sfield, &ires[0]);
} else {
ufunc->evaluate(srcrend, elem, sfield, npts, &pcoord[0], &ires[0]);
}
// Copy data to nodes
for (UInt n = 0; n < npts; n++) {
const MeshObj &node = *sres.nodes[n].node;
for (UInt d = 0; d < dfield.dim(node); d++)
((double*)dfield.data(node))[d] = ires[n*dfield.dim(node)+d];
}
} // for fields
} // for searchresult
}
void CommReg::SyncAttributes(UInt obj_type) {
Trace __trace("CommReg::SyncAttributes(UInt obj_type)");
if (obj_type & MeshObj::NODE) sync(node_rel);
if (obj_type & MeshObj::EDGE) sync(edge_rel);
if (obj_type & MeshObj::FACE) sync(face_rel);
}
void static sync(CommRel &comm) {
Trace __trace("sync(CommRel &comm)");
UInt csize = Par::Size();
SparseMsg msg;
// Sizing loop
std::vector<UInt> send_sizes_all(csize, 0);
std::vector<UInt> to_proc;
std::vector<UInt> to_sizes;
CommRel::MapType::iterator di = comm.domain_begin(), de = comm.domain_end();
for (; di != de; ++di) {
UInt proc = di->processor;
std::vector<UInt>::iterator lb =
std::lower_bound(to_proc.begin(), to_proc.end(), proc);
if (lb == to_proc.end() || *lb != proc)
to_proc.insert(lb, proc);
// Attr
send_sizes_all[proc] += SparsePack<Attr>::size();
} //sizes
UInt nsend = to_proc.size();
msg.setPattern(nsend, nsend == 0 ? NULL : &to_proc[0]);
to_sizes.resize(nsend, 0);
for (UInt i = 0; i < nsend; i++)
to_sizes[i] = send_sizes_all[to_proc[i]];
msg.setSizes(nsend == 0 ? NULL : &to_sizes[0]);
// Pack loop
di = comm.domain_begin();
for (; di != de; ++di) {
UInt proc = di->processor;
MeshObj &obj = *di->obj;
SparseMsg::buffer &b = *msg.getSendBuffer(proc);
// Attr
SparsePack<Attr>(b, GetAttr(obj));
}
if (!msg.filled())
Throw() << "Message not full in sync attr!";
msg.communicate();
// Create an object to context map so that objects are
// only updated in the mesh onces.
typedef std::map<MeshObj*, Context> Obj_To_Ctxt_Type;
Obj_To_Ctxt_Type obj_to_ctxt;
// Unpack
di = comm.domain_begin();
for (; di != de; ++di) {
MeshObj &obj = *di->obj;
UInt proc = di->processor;
SparseMsg::buffer &b = *msg.getRecvBuffer(proc);
Attr a;
SparseUnpack<Attr>(b, a);
// Now the kernel. Or all attributes except the shared ones
// (ownership, etc...)
const Attr &oa = GetAttr(obj);
// For sanity:
ThrowRequire(a.GetType() == oa.GetType());
ThrowRequire(a.GetBlock() == oa.GetBlock());
// Now merge the contexts
Context c = a.GetContext();
const Context &oc = oa.GetContext();
Context nc(oc);
// More sanity checks
//ThrowRequire(c.is_set(Attr::ACTIVE_ID) == oc.is_set(Attr::ACTIVE_ID));
if (!(c.is_set(Attr::ACTIVE_ID) == oc.is_set(Attr::ACTIVE_ID))) {
Par::Out() << "Error, ACTIVE_ID incongruence, obj:" << obj;
Par::Out() << "Incoming ctxt:" << c << std::endl;
Throw();
}
ThrowRequire(c.is_set(Attr::SHARED_ID) && oc.is_set(Attr::SHARED_ID));
ThrowRequire(c.is_set(Attr::GENESIS_ID) == oc.is_set(Attr::GENESIS_ID));
// Both can't claim to own object
//ThrowRequire(!(c.is_set(Attr::OWNED_ID) && oc.is_set(Attr::OWNED_ID)));
if ((c.is_set(Attr::OWNED_ID) && oc.is_set(Attr::OWNED_ID))) {
Par::Out() << "Error, OWNED_ID incongruence, obj:" << obj;
Par::Out() << "Incoming attr:" << a << std::endl;
Par::Out() << "From processor:" << proc << std::endl;
Throw();
}
// Clear the bits not to merge
c.clear(Attr::SHARED_ID);
c.clear(Attr::OWNED_ID);
c.clear(Attr::ACTIVE_ID);
c.clear(Attr::GENESIS_ID);
// Or the rest
nc |= c;
// Add or OR in the new context, depending on whether object is in map.
std::pair<Obj_To_Ctxt_Type::iterator, bool> otci =
obj_to_ctxt.insert(std::make_pair(&obj, nc));
if (otci.second == false) { // already there
otci.first->second |= nc;
}
}
// One last loop through the object map, updating the mesh
Obj_To_Ctxt_Type::iterator oi = obj_to_ctxt.begin(), oe = obj_to_ctxt.end();
for (; oi != oe; ++oi) {
MeshObj &obj = *oi->first;
Context &nc = oi->second;
if (nc != GetMeshObjContext(obj)) {
Attr oa(GetAttr(obj), nc);
comm.DomainMesh()->update_obj(&obj, oa);
}
}
}
bool
proc_probe(pid_t pid, int opt, proc_t * const pproc)
{
FUNC_BEGIN("%d,%d,%p", pid, opt, pproc);
assert(pproc);
if (pproc == NULL)
{
FUNC_RET("%d", false);
}
#ifdef HAVE_PROCFS
/* Grab preliminary information from procfs */
if (!check_procfs(pid))
{
WARNING("procfs entries missing or invalid");
FUNC_RET("%d", false);
}
char buffer[4096];
sprintf(buffer, PROCFS "/%d/stat", pid);
int fd = open(buffer, O_RDONLY);
if (fd < 0)
{
WARNING("procfs stat missing or unaccessable");
FUNC_RET("%d", false);
}
int len = read(fd, buffer, sizeof(buffer) - 1);
close(fd);
if (len < 0)
{
WARNING("failed to grab stat from procfs");
FUNC_RET("%d", false);
}
buffer[len] = '\0';
/* Extract interested information */
struct tms tm;
int offset = 0;
char * token = buffer;
do
{
switch (offset++)
{
case 0: /* pid */
sscanf(token, "%d", &pproc->pid);
break;
case 1: /* comm */
break;
case 2: /* state */
sscanf(token, "%c", &pproc->state);
break;
case 3: /* ppid */
sscanf(token, "%d", &pproc->ppid);
break;
case 4: /* pgrp */
case 5: /* session */
case 6: /* tty_nr */
case 7: /* tty_pgrp */
break;
case 8: /* flags */
sscanf(token, "%lu", &pproc->flags);
break;
case 9: /* min_flt */
sscanf(token, "%lu", &pproc->minflt);
break;
case 10: /* cmin_flt */
sscanf(token, "%lu", &pproc->cminflt);
break;
case 11: /* maj_flt */
sscanf(token, "%lu", &pproc->majflt);
break;
case 12: /* cmaj_flt */
sscanf(token, "%lu", &pproc->cmajflt);
break;
case 13: /* utime */
sscanf(token, "%lu", &tm.tms_utime);
break;
case 14: /* stime */
sscanf(token, "%lu", &tm.tms_stime);
break;
case 15: /* cutime */
sscanf(token, "%ld", &tm.tms_cutime);
break;
case 16: /* cstime */
sscanf(token, "%ld", &tm.tms_cstime);
break;
case 17: /* priority */
case 18: /* nice */
case 19: /* num_threads (since 2.6 kernel) */
case 20: /* it_real_value */
case 21: /* start_time */
break;
case 22: /* vsize */
sscanf(token, "%lu", &pproc->vsize);
break;
case 23: /* rss */
sscanf(token, "%ld", &pproc->rss);
break;
case 24: /* rlim_rss */
case 25: /* start_code */
sscanf(token, "%lu", &pproc->start_code);
break;
case 26: /* end_code */
sscanf(token, "%lu", &pproc->end_code);
break;
case 27: /* start_stack */
sscanf(token, "%lu", &pproc->start_stack);
break;
case 28: /* esp */
case 29: /* eip */
case 30: /* pending_signal */
case 31: /* blocked_signal */
case 32: /* sigign */
case 33: /* sigcatch */
case 34: /* wchan */
case 35: /* nswap */
case 36: /* cnswap */
case 37: /* exit_signal */
case 38: /* processor */
case 39: /* rt_priority */
case 40: /* policy */
default:
break;
}
} while (strsep(&token, " ") != NULL);
long CLK_TCK = sysconf(_SC_CLK_TCK);
#define TS_UPDATE_CLK(pts,clk) \
{{{ \
(pts)->tv_sec = ((time_t)((clk) / CLK_TCK)); \
(pts)->tv_nsec = (1000000000UL * ((clk) % (CLK_TCK)) / CLK_TCK ); \
}}} /* TS_UPDATE_CLK */
TS_UPDATE_CLK(&pproc->utime, tm.tms_utime);
TS_UPDATE_CLK(&pproc->stime, tm.tms_stime);
TS_UPDATE_CLK(&pproc->cutime, tm.tms_cutime);
TS_UPDATE_CLK(&pproc->cstime, tm.tms_cstime);
DBG("proc.pid % 10d", pproc->pid);
DBG("proc.ppid % 10d", pproc->ppid);
DBG("proc.state %c", pproc->state);
DBG("proc.flags 0x%016lx", pproc->flags);
DBG("proc.utime %010lu", ts2ms(pproc->utime));
DBG("proc.stime %010lu", ts2ms(pproc->stime));
DBG("proc.cutime % 10ld", ts2ms(pproc->cutime));
DBG("proc.cstime % 10ld", ts2ms(pproc->cstime));
DBG("proc.minflt %010lu", pproc->minflt);
DBG("proc.cminflt %010lu", pproc->cminflt);
DBG("proc.majflt %010lu", pproc->majflt);
DBG("proc.cmajflt %010lu", pproc->cmajflt);
DBG("proc.vsize %010lu", pproc->vsize);
DBG("proc.rss % 10ld", pproc->rss);
#else
#warning "proc_probe() requires procfs"
#endif /* HAVE_PROCFS */
trace_proxy_t __trace = TRACE_PROXY(pproc);
/* Inspect process registers */
if (opt & PROBE_REGS)
{
if (__trace(T_GETREGS, NULL, pproc) != 0)
{
FUNC_RET("%d", false);
}
#ifdef __x86_64__
DBG("regs.r15 0x%016lx", pproc->regs.r15);
DBG("regs.r14 0x%016lx", pproc->regs.r14);
DBG("regs.r13 0x%016lx", pproc->regs.r13);
DBG("regs.r12 0x%016lx", pproc->regs.r12);
DBG("regs.rbp 0x%016lx", pproc->regs.rbp);
DBG("regs.rbx 0x%016lx", pproc->regs.rbx);
DBG("regs.r11 0x%016lx", pproc->regs.r11);
DBG("regs.r10 0x%016lx", pproc->regs.r10);
DBG("regs.r9 0x%016lx", pproc->regs.r9);
DBG("regs.r8 0x%016lx", pproc->regs.r8);
DBG("regs.rax 0x%016lx", pproc->regs.rax);
DBG("regs.rcx 0x%016lx", pproc->regs.rcx);
DBG("regs.rdx 0x%016lx", pproc->regs.rdx);
DBG("regs.rsi 0x%016lx", pproc->regs.rsi);
DBG("regs.rdi 0x%016lx", pproc->regs.rdi);
DBG("regs.orig_rax 0x%016lx", pproc->regs.orig_rax);
DBG("regs.rip 0x%016lx", pproc->regs.rip);
DBG("regs.cs 0x%016lx", pproc->regs.cs);
DBG("regs.eflags 0x%016lx", pproc->regs.eflags);
DBG("regs.rsp 0x%016lx", pproc->regs.rsp);
DBG("regs.ss 0x%016lx", pproc->regs.ss);
DBG("regs.fs_base 0x%016lx", pproc->regs.fs_base);
DBG("regs.gs_base 0x%016lx", pproc->regs.gs_base);
DBG("regs.ds 0x%016lx", pproc->regs.ds);
DBG("regs.es 0x%016lx", pproc->regs.es);
DBG("regs.fs 0x%016lx", pproc->regs.fs);
DBG("regs.gs 0x%016lx", pproc->regs.gs);
#else /* __i386__ */
DBG("regs.ebx 0x%016lx", pproc->regs.ebx);
DBG("regs.ecx 0x%016lx", pproc->regs.ecx);
DBG("regs.edx 0x%016lx", pproc->regs.edx);
DBG("regs.esi 0x%016lx", pproc->regs.esi);
DBG("regs.edi 0x%016lx", pproc->regs.edi);
DBG("regs.ebp 0x%016lx", pproc->regs.ebp);
DBG("regs.eax 0x%016lx", pproc->regs.eax);
DBG("regs.xds 0x%016lx", pproc->regs.xds);
DBG("regs.xes 0x%016lx", pproc->regs.xes);
DBG("regs.xfs 0x%016lx", pproc->regs.xfs);
DBG("regs.xgs 0x%016lx", pproc->regs.xgs);
DBG("regs.orig_eax 0x%016lx", pproc->regs.orig_eax);
DBG("regs.eip 0x%016lx", pproc->regs.eip);
DBG("regs.xcs 0x%016lx", pproc->regs.xcs);
DBG("regs.eflags 0x%016lx", pproc->regs.eflags);
DBG("regs.esp 0x%016lx", pproc->regs.esp);
DBG("regs.xss 0x%016lx", pproc->regs.xss);
#endif /* __x86_64__ */
}
/* Inspect current instruction */
if (opt & PROBE_OP)
{
#ifdef __x86_64__
pproc->op = pproc->regs.rip;
#else /* __i386__ */
pproc->op = pproc->regs.eip;
#endif /* __x86_64__ */
if (!pproc->tflags.single_step)
{
pproc->op -= 2; // shift back 16bit in syscall tracing mode
}
if (__trace(T_GETDATA, (long *)&pproc->op, pproc) != 0)
{
FUNC_RET("%d", false);
}
DBG("proc.op 0x%016lx", pproc->op);
}
/* Inspect signal information */
if (opt & PROBE_SIGINFO)
{
if (__trace(T_GETSIGINFO, NULL, pproc) != 0)
{
FUNC_RET("%d", false);
}
DBG("proc.siginfo.si_signo % 10d", pproc->siginfo.si_signo);
DBG("proc.siginfo.si_errno % 10d", pproc->siginfo.si_errno);
DBG("proc.siginfo.si_code % 10d", pproc->siginfo.si_code);
}
FUNC_RET("%d", true);
}
/**
* Called in state CHANGE to process a ping.
*
* \param[in] ping Ping to process
*/
static void
state_change_process_ping(const sup_ping_t *ping)
{
switch (ping->view.state)
{
case SUP_STATE_UNKNOWN:
/* Not possible */
break;
case SUP_STATE_CHANGE:
if (self_is_coord())
{
__trace("i'm coord");
/* Go to state ACCEPT if all the nodes in our clique see us as
* their coordinator. This will initiate the agreement phase.
*/
if (clique_sees_self_as_coord())
{
__trace("+++ EVERYONE IN MY CLIQUE SEES ME AS COORD");
accept_clique(next_gen());
__trace("\t=> new state=%s, accepted=%u",
sup_state_name(self->view.state),
self->view.accepted);
}
}
break;
case SUP_STATE_ACCEPT:
if (!self_is_coord())
{
/* Go to state ACCEPT if the sender is our coordinator, views the
* same clique as us and its accepted clique generation is higher
* than ours.
*/
if (ping->sender == self->view.coord
&& exa_nodeset_equals(&ping->view.clique, &self->view.clique)
&& ping->view.accepted > self_highest_gen())
{
__trace("coord ok, view ok, accepted ok => i accept");
accept_clique(ping->view.accepted);
}
}
break;
case SUP_STATE_COMMIT:
if (!self_is_coord())
{
/* Commit the membership if we accepted it */
if (exa_nodeset_equals(&ping->view.clique, &accepted_clique)
&& ping->view.committed == self->view.accepted
&& ping->view.committed > self->view.committed)
{
__trace("saw commit %u from %u and i accepted %u => i commit %u",
ping->view.committed, ping->sender, self->view.accepted,
self->view.accepted);
commit_clique();
}
}
break;
}
}
//BOPI
// !IROUTINE: SpaceDir
//
// !INTERFACE:
SpaceDir::SpaceDir(
//
// !RETURN VALUE:
// Pointer to a new SpaceDir
//
// !ARGUMENTS:
double _proc_min[3], // min of objects in otree for this proc
double _proc_max[3], // max of objects in otree for this proc
OTree *_otree // Otree of objects on this proc
// NOTE: _otree should be commited before being passed in
// NOTE: SpaceDir won't destruct this otree
){
//
// !DESCRIPTION:
// Construct SpaceDir
//EOPI
//-----------------------------------------------------------------------------
Trace __trace("SpaceDir::SpaceDir()");
// Set variables
otree=_otree;
proc_min[0]=_proc_min[0];
proc_min[1]=_proc_min[1];
proc_min[2]=_proc_min[2];
proc_max[0]=_proc_max[0];
proc_max[1]=_proc_max[1];
proc_max[2]=_proc_max[2];
// EVENTUALLY NEED TO MAKE THIS MORE SCALABLE, BUT FOR
// AN INITIAL TEST DO SOMETHING EASY
// Distribute min-max box of each proc
// Pack up local minmax
std::vector<double> local_minmax(6,0.0);
local_minmax[0]=proc_min[0];
local_minmax[1]=proc_min[1];
local_minmax[2]=proc_min[2];
local_minmax[3]=proc_max[0];
local_minmax[4]=proc_max[1];
local_minmax[5]=proc_max[2];
// Number of procs
int num_procs = Par::Size();
if (num_procs <1) Throw() << "Error: number of procs is less than 1";
// Allocate buffer for global minmax list
std::vector<double> global_minmax(6*num_procs, 0.0);
// Do all gather to get info from all procs
MPI_Allgather(&local_minmax[0], 6, MPI_DOUBLE, &global_minmax[0], 6, MPI_DOUBLE, Par::Comm());
// Setup a list of numbers to store in proc tree
proc_nums=new int[num_procs];
for (int i=0; i<num_procs; i++) {
proc_nums[i]=i;
}
// Construct tree
proc_otree = new OTree(num_procs);
// Add proc min max boxes
for (int i=0; i<num_procs; i++) {
// Get min and max from global list
double *min=&global_minmax[6*i];
double *max=&global_minmax[6*i+3];
// Add proc to tree
proc_otree->add(min, max, (void *)(proc_nums+i));
}
// Commit tree to make it usable
proc_otree->commit();
}
