/////////////////////////////////////////////////////////////////////////
// Vertex Loader (used to read images and load the vertex data of the graph)
//bool vertex_loader(graph_type& graph, const std::string& fname,
// const std::string& line)
bool vertex_loader(graphlab::distributed_control& dc, graph_type& graph, string img_path)
{
// force a "/" at the end of the path
// make sure to check that the path is non-empty. (you do not
// want to make the empty path "" the root path "/" )
string path = img_path;
if (path.length() > 0 && path[path.length() - 1] != '/') path = path + "/";
vector<string> graph_files;
string search_prefix;
graphlab::fs_util::list_files_with_prefix(path, search_prefix, graph_files);
if (graph_files.size() == 0)
logstream(LOG_WARNING) << "No files found in " << path << std::endl;
// vertex data & id
graphlab::vertex_id_type vid(-1);
///////////////////////////////////////////////////////
// Loop over files
for(size_t i = 0; i < graph_files.size(); ++i)
{
// Each machine loads corresponding file
if (i % dc.numprocs() == dc.procid())
{
if (opts.verbose > 0)
logstream(LOG_EMPH)
<< "Process: " << dc.procid() << "/" << dc.numprocs() << " "
<< "picked image: " << graph_files[i] << "\n";
vid = i;
vertex_data vdata;
vdata.empty = false;
vdata.img_path = graph_files[i];
vdata.features.img_idx = i;
graph.add_vertex(vid, vdata);
}
}
return true;
}
/////////////////////////////////////////////////////////////////////////
// Load the UAI file. Each factor as a different vertex
void loadUAIfile(graphlab::distributed_control& dc, graph_type& graph, string graph_file)
{
// Not sure why this is needed
dc.barrier();
// Open file
ifstream in(graph_file.c_str());
//CHECK(in.good(),"Could not open file: "+graph_file);
CHECK(in.good());
// Read type of network
string name;
in >> name;
//CHECK(name.compare("MARKOV")==0, "Only Markov networks are supported. Are you sure this is a typeUAI energy file?");
CHECK(name.compare("MARKOV")==0);
// Read size of graph
int nnodes, nfactors;
in >> nnodes;
//CHECK(nnodes>0, "No. of nodes can't be negative. Are you sure this is a typeUAI energy file?");
CHECK(nnodes>0);
// Read node cardinalities
vector<int> cardinalities(nnodes,0);
int cardinality_i, sum_of_cardinalities = 0;
for (int i = 0; i != nnodes; ++i)
{
in >> cardinality_i;
cardinalities[i] = cardinality_i;
sum_of_cardinalities += cardinality_i;
//cout << cardinalities[i] << " ";
//CHECK(in.good(), "Could not finish reading cardinalities. Are you sure this is a typeUAI energy file?");
CHECK(in.good());
}
// Read no. of factors
in >> nfactors;
//factor_size.resize(nfactors); factor_id.resize(nfactors);
vector<int> factor_size(nfactors,0); //vector<int> factor_id(nfactors,0);
vector< vector<int> > factor_memb; factor_memb.resize(nfactors);
int temp1, temp2;
// Loop and read factor members
for (int i=0; i!=nfactors; ++i)
{
in >> temp1;
factor_size[i] = temp1;
factor_memb[i].resize(temp1);
for (int j=0; j!=temp1; ++j)
{
in >> temp2;
factor_memb[i][j] = temp2;
}
//CHECK(in.good(), "Could not finish reading cardinalities. Are you sure this is a typeUAI energy file?");
CHECK(in.good());
}
if (opts.verbose > 0)
cout
<< "Finished Reading UAI-Preamble:"
<< " #Nodes = " << nnodes
<< ", #Factors = "<< nfactors
<< ", Average Cardinality = " << double(sum_of_cardinalities)/nfactors
<< "\n";
// Now read factor potentials
for (int i=0; i!=nfactors; ++i)
{
int cardprod; double potential_value; //, energy;
in >> cardprod;
vertex_data vdata;
vdata.nvars = factor_size[i];
if (vdata.nvars > 1) {
vdata.degree = vdata.nvars; // Factor degree.
}
vdata.cards.resize(factor_size[i]);
vdata.neighbors.resize(factor_size[i]);
vector<edge_data> edata(factor_size[i]);
int cardprod2 = 1;
for (int j=0; j!=factor_size[i]; ++j)
{
vdata.cards[j] = cardinalities[factor_memb[i][j]];
vdata.neighbors[j] = factor_memb[i][j]; // afm (check if this was intended!)
cardprod2 *= vdata.cards[j];
// Also create edge structs here
if (factor_size[i]>1)
{
edata[j].varid = factor_memb[i][j];
edata[j].card = cardinalities[edata[j].varid];
edata[j].multiplier_messages.setZero(edata[j].card);
edata[j].local_messages.setZero(edata[j].card);
}
}
//CHECK_EQ(cardprod, cardprod2, "Incorrectly sized factor");
CHECK_EQ(cardprod, cardprod2);
// Read factor potentials
vdata.potentials.resize(cardprod);
for (int k = 0; k != cardprod; ++k)
{
in >> potential_value;
//energy = Potential2Energy(potential_value);
vdata.potentials[k] = log10(potential_value);
}
//CHECK(in.good(), "Could not finish reading factor tables. Are you sure this is a typeUAI energy file?");
CHECK(in.good());
// allocate factors evenly to different machines.
if (i%dc.numprocs() != dc.procid())
continue;
// If all is well, add vertex and edges
graph.add_vertex(i,vdata);
if (factor_size[i] > 1) // if not a unary, add edges to unaries
for (int j=0; j!=factor_size[i]; ++j)
graph.add_edge(i,edata[j].varid,edata[j]);
if (opts.verbose > 1)
{
cout << "Machine #" << dc.procid() << ", Vertex Id = " << i
<< " with " << vdata.nvars << " variables.";
if (factor_size[i] > 1)
{
cout << ", Edges = ";
for (int j=0; j!=factor_size[i]; ++j)
cout << ", (" << i << "," << edata[j].varid << ")";
}
cout << "\n";
cout << "potential: " << vdata.potentials << "\n";
}
} // End of reading factors
dc.barrier();
} // end of loading UAI file
bool graph_loader(graphlab::distributed_control& dc, graph_type& graph, string img_dir)
{
// force a "/" at the end of the path
// make sure to check that the path is non-empty. (you do not
// want to make the empty path "" the root path "/" )
string path = img_dir;
if (path.length() > 0 && path[path.length() - 1] != '/') path = path + "/";
vector<string> graph_files;
string search_prefix;
graphlab::fs_util::list_files_with_prefix(path, search_prefix, graph_files);
if (graph_files.size() == 0)
logstream(LOG_WARNING) << "No files found in " << path << std::endl;
if (opts.verbose > 2)
logstream(LOG_EMPH)
<< "Total number of images: " << graph_files.size() << "\n";
// vertex data & id
graphlab::vertex_id_type vid(-1);
graphlab::vertex_id_type other_vid;
///////////////////////////////////////////////////////
// Loop over files
for(size_t i = 0; i < graph_files.size(); ++i)
{
// Each machine loads corresponding file
if (i % dc.numprocs() == dc.procid())
{
if (opts.verbose > 0)
logstream(LOG_EMPH)
<< "Process: " << dc.procid() << "/" << dc.numprocs() << " "
<< "picked image: " << graph_files[i] << "\n";
vid = i;
vertex_data vdata;
vdata.empty = false;
vdata.img_path = graph_files[i];
vdata.features.img_idx = i;
graph.add_vertex(vid, vdata);
if (opts.verbose > 2)
logstream(LOG_EMPH)
<< "Vertex " << i << " Image: " << vdata.img_path << "\n";
}
}
// Adding edges between every pair of vertices to create a fully connected graph
// no duplicate edges are added
for(size_t i = 0; i < graph_files.size()-1; ++i)
{
vid = i;
for(size_t j = i+1; j < graph_files.size(); ++j)
{
other_vid = j;
if (opts.verbose > 0)
logstream(LOG_EMPH) << "Adding edge: (" << vid << "," << other_vid << ")\n";
edge_data edata; edata.empty = false;
graph.add_edge(vid,other_vid,edata);
}
}
return true;
}
/////////////////////////////////////////////////////////////////////////
// Load the UAI file. Each factor as a different vertex
void loadUAIfile(graphlab::distributed_control& dc, graph_type& graph, string graph_file, int& nodes)
{
// Not sure why this is needed
dc.barrier();
// Open file
ifstream in(graph_file.c_str());
//CHECK(in.good(),"Could not open file: "+graph_file);
CHECK(in.good());
// Read type of network
string name;
in >> name;
//CHECK(name.compare("MARKOV")==0, "Only Markov networks are supported. Are you sure this is a typeUAI energy file?");
CHECK(name.compare("MARKOV")==0);
// Read size of graph
int nnodes, nfactors;
in >> nnodes;
nodes = nnodes;
//CHECK(nnodes>0, "No. of nodes can't be negative. Are you sure this is a typeUAI energy file?");
CHECK(nnodes>0);
// Read node cardinalities
vector<int> cardinalities(nnodes,0);
int cardinality_i, sum_of_cardinalities = 0;
for (int i = 0; i != nnodes; ++i)
{
in >> cardinality_i;
cardinalities[i] = cardinality_i;
sum_of_cardinalities += cardinality_i;
//CHECK(in.good(), "Could not finish reading cardinalities. Are you sure this is a typeUAI energy file?");
CHECK(in.good());
}
int vid = 0;
if(opts.algorithm != 0){
for(int i = 0; i < nnodes; i++){ //temporary .. put condition
vertex_data vdata;
vdata.factor_type = VAR;
vdata.nvars = 1;
vdata.cards.resize(1, cardinalities[i]);
vdata.potentials.setZero(cardinalities[i]);
vdata.beliefs.setConstant(cardinalities[i], 0.5);
graph.add_vertex(vid, vdata);
vid++;
}
}
// Read no. of factors
in >> nfactors;
//factor_size.resize(nfactors); factor_id.resize(nfactors);
vector<int> factor_size(nfactors,0); //vector<int> factor_id(nfactors,0);
vector< vector<int> > factor_memb; factor_memb.resize(nfactors);
int temp1, temp2;
// Loop and read factor members
for (int i=0; i!=nfactors; ++i)
{
in >> temp1;
factor_size[i] = temp1;
factor_memb[i].resize(temp1);
for (int j=0; j!=temp1; ++j)
{
in >> temp2;
factor_memb[i][j] = temp2;
}
//CHECK(in.good(), "Could not finish reading cardinalities. Are you sure this is a typeUAI energy file?");
CHECK(in.good());
}
if (opts.verbose > 1)
cout
<< "Finished Reading UAI-Preamble:"
<< " #Nodes = " << nnodes
<< ", #Factors = "<< nfactors
<< ", Average Cardinality = " << double(sum_of_cardinalities)/nfactors
<< "\n";
// Now read factor potentials
for (int i=0; i!=nfactors; ++i)
{
int cardprod; double potential_value; //, energy;
in >> cardprod;
vertex_data vdata;
vdata.nvars = factor_size[i];
if (vdata.nvars > 1) {
vdata.degree = vdata.nvars; // Factor degree.
vdata.factor_type = DENSE;
}
else {
vdata.degree = 1; // Factor degree.
vdata.factor_type = XOR;
}
vdata.cards.resize(factor_size[i]);
vdata.neighbors.resize(factor_size[i]);
vector<edge_data> edata(factor_size[i]);
vector<int> varid(factor_size[i]);
vector<int> card(factor_size[i]);
int cardprod2 = 1;
for (int j=0; j!=factor_size[i]; ++j)
{
vdata.cards[j] = cardinalities[factor_memb[i][j]];
vdata.neighbors[j] = factor_memb[i][j]; // afm (check if this was intended!)
cardprod2 *= vdata.cards[j];
// Also create edge structs here
//if (factor_size[i]>1)
// {
varid[j] = factor_memb[i][j];
card[j] = cardinalities[varid[j]];
edata[j].multiplier_messages.setZero(card[j]);
edata[j].local_messages.setZero(card[j]);
edata[j].potentials.setZero(card[j]);
// }
}
//CHECK_EQ(cardprod, cardprod2, "Incorrectly sized factor");
CHECK_EQ(cardprod, cardprod2);
// Read factor potentials
vdata.potentials.resize(cardprod);
vdata.beliefs.resize(cardprod);
int x_offset = 0;
for(int x=0; x< vdata.nvars; x++){
for(int y=0; y<vdata.cards[x]; y++){
vdata.beliefs[x_offset+y] = 1.0/vdata.cards[x];
}
x_offset += vdata.cards[x];
}
vdata.factor_beliefs.setConstant(cardprod, 1.0/cardprod);
for (int k = 0; k != cardprod; ++k)
{
in >> potential_value;
//energy = Potential2Energy(potential_value);
vdata.potentials[k] = log10(potential_value) ;
}
//CHECK(in.good(), "Could not finish reading factor tables. Are you sure this is a typeUAI energy file?");
CHECK(in.good());
vdata.potentials.maxCoeff(&vdata.best_configuration);
// allocate factors evenly to different machines.
if (i%dc.numprocs() != dc.procid())
continue;
// If all is well, add vertex and edge
graph.add_vertex(vid ,vdata);
if (factor_size[i] > 1 || opts.algorithm > 0) // if not a unary, add edges to unaries
for (int j=0; j!=factor_size[i]; ++j)
graph.add_edge(vid,varid[j],edata[j]);
//after adding everything increment vertex id
vid++;
if (opts.verbose > 1)
{
cout << "Machine #" << dc.procid() << ", Vertex Id = " << i
<< " with " << vdata.nvars << " variables.";
if (factor_size[i] > 1)
{
cout << ", Edges = ";
for (int j=0; j!=factor_size[i]; ++j)
cout << ", (" << i << "," << varid[j] << ")";
}
cout << "\n";
cout << "potential: " << vdata.potentials << "\n";
}
} // End of reading factors
dc.barrier();
} // end of loading UAI file