void main()
{
//ClassMap.addMap( &ClassMap::setIdade,&ClassMap::getIdade,"Coluna");
//ClassMap::addMap( void (*) (int),&ClassMap::getIdade,"Coluna");
void (*pFuncTeste) (int) = &ClassMap::setIdade;
void (ClassMap::*pFunc) (int) = &ClassMap::setIdade;
ClassMap objeto;
(objeto.*pFunc)(30);
cout << objeto.getIdade() << endl;
getch();
return;
}
shark::ClassificationDataset transformToClassificationDataset(
const shark::LabeledData<shark::RealVector, shark::RealVector>& data,
ClassMap& classMap)
{
using namespace shark;
std::vector<shark::RealVector> inputs;
std::vector<unsigned int> labels;
auto elements = data.elements();
for (auto iter = elements.begin(); iter != elements.end(); ++iter) {
inputs.push_back(iter->input);
labels.push_back(classMap.valueToClass(iter->label[0]));
classMap.optimizeClass(iter->label[0]);
}
shark::ClassificationDataset result = createLabeledDataFromRange(inputs, labels);
return std::move(result);
}
void ServerConfig::CrossCheckConnectBlocks(ServerConfig* current)
{
typedef std::map<std::string, ConnectClass*> ClassMap;
ClassMap oldBlocksByMask;
if (current)
{
for(ClassVector::iterator i = current->Classes.begin(); i != current->Classes.end(); ++i)
{
ConnectClass* c = *i;
if (c->name.substr(0, 8) != "unnamed-")
{
oldBlocksByMask["n" + c->name] = c;
}
else if (c->type == CC_ALLOW || c->type == CC_DENY)
{
std::string typeMask = (c->type == CC_ALLOW) ? "a" : "d";
typeMask += c->host;
oldBlocksByMask[typeMask] = c;
}
}
}
int blk_count = config_data.count("connect");
if (blk_count == 0)
{
// No connect blocks found; make a trivial default block
std::vector<KeyVal>* items;
ConfigTag* tag = ConfigTag::create("connect", "<auto>", 0, items);
items->push_back(std::make_pair("allow", "*"));
config_data.insert(std::make_pair("connect", tag));
blk_count = 1;
}
Classes.resize(blk_count);
std::map<std::string, int> names;
bool try_again = true;
for(int tries=0; try_again; tries++)
{
try_again = false;
ConfigTagList tags = ConfTags("connect");
int i=0;
for(ConfigIter it = tags.first; it != tags.second; ++it, ++i)
{
ConfigTag* tag = it->second;
if (Classes[i])
continue;
ConnectClass* parent = NULL;
std::string parentName = tag->getString("parent");
if (!parentName.empty())
{
std::map<std::string,int>::iterator parentIter = names.find(parentName);
if (parentIter == names.end())
{
try_again = true;
// couldn't find parent this time. If it's the last time, we'll never find it.
if (tries >= blk_count)
throw CoreException("Could not find parent connect class \"" + parentName + "\" for connect block at " + tag->getTagLocation());
continue;
}
parent = Classes[parentIter->second];
}
std::string name = tag->getString("name");
std::string mask, typeMask;
char type;
if (tag->readString("allow", mask, false))
{
type = CC_ALLOW;
typeMask = 'a' + mask;
}
else if (tag->readString("deny", mask, false))
{
type = CC_DENY;
typeMask = 'd' + mask;
}
else if (!name.empty())
{
type = CC_NAMED;
mask = name;
typeMask = 'n' + mask;
}
else
{
throw CoreException("Connect class must have allow, deny, or name specified at " + tag->getTagLocation());
}
if (name.empty())
{
name = "unnamed-" + ConvToStr(i);
}
else
{
typeMask = 'n' + name;
}
if (names.find(name) != names.end())
throw CoreException("Two connect classes with name \"" + name + "\" defined!");
names[name] = i;
ConnectClass* me = parent ?
new ConnectClass(tag, type, mask, *parent) :
new ConnectClass(tag, type, mask);
me->name = name;
me->registration_timeout = tag->getInt("timeout", me->registration_timeout);
me->pingtime = tag->getInt("pingfreq", me->pingtime);
std::string sendq;
if (tag->readString("sendq", sendq))
{
// attempt to guess a good hard/soft sendq from a single value
long value = atol(sendq.c_str());
if (value > 16384)
me->softsendqmax = value / 16;
else
me->softsendqmax = value;
me->hardsendqmax = value * 8;
}
me->softsendqmax = tag->getInt("softsendq", me->softsendqmax);
me->hardsendqmax = tag->getInt("hardsendq", me->hardsendqmax);
me->recvqmax = tag->getInt("recvq", me->recvqmax);
me->penaltythreshold = tag->getInt("threshold", me->penaltythreshold);
me->commandrate = tag->getInt("commandrate", me->commandrate);
me->fakelag = tag->getBool("fakelag", me->fakelag);
me->maxlocal = tag->getInt("localmax", me->maxlocal);
me->maxglobal = tag->getInt("globalmax", me->maxglobal);
me->maxchans = tag->getInt("maxchans", me->maxchans);
me->maxconnwarn = tag->getBool("maxconnwarn", me->maxconnwarn);
me->limit = tag->getInt("limit", me->limit);
me->nouserdns = tag->getBool("nouserdns", me->nouserdns);
ClassMap::iterator oldMask = oldBlocksByMask.find(typeMask);
if (oldMask != oldBlocksByMask.end())
{
ConnectClass* old = oldMask->second;
oldBlocksByMask.erase(oldMask);
old->Update(me);
delete me;
me = old;
}
Classes[i] = me;
}
}
}
/**
* Main routine for the controller.
*/
int mainController() {
NTriangulation* t;
// Start logging.
logger.open(logFile);
if (! logger) {
fprintf(stderr, "Could not open %s for writing.\n", logFile);
return 1;
}
// Do it.
for (NPacket* p = tree; p; p = p->nextTreePacket())
if (p->type() == PACKET_TRIANGULATION) {
nTris++;
ctrlFarmTri(static_cast<NTriangulation*>(p));
}
// Kill off any slaves that never started working, since there are no
// tasks left to give them.
if (nRunningSlaves < nSlaves)
for (int i = nRunningSlaves; i < nSlaves; i++)
ctrlStopSlave(i + 1);
// Wait for remaining slaves to finish.
while (nRunningSlaves > 0)
ctrlFarmTri(0);
// Done!
ctrlLogStamp() << "All slaves finished." << std::endl;
// Write the summary of results.
if (nClasses) {
printf("EQUIVALENCE CLASSES:\n\n");
if (outFile) {
newTree = new NContainer();
newTree->setLabel("Equivalence Classes");
}
int classNum = 1;
std::string className;
NContainer* classCnt = 0;
ClassMap::iterator cit, cit2;
int c;
for (cit = eClass.begin(); cit != eClass.end(); cit++)
if (cit->second >= 0) {
// The first triangulation of a new equivalence class.
c = cit->second;
std::ostringstream s;
s << "Class " << classNum << " : " <<
cit->first->homology().str();
className = s.str();
classNum++;
printf("%s\n\n", className.c_str());
if (outFile) {
classCnt = new NContainer();
classCnt->setLabel(className);
newTree->insertChildLast(classCnt);
}
// Find the triangulations in this class, and erase the
// class as we go.
for (cit2 = cit; cit2 != eClass.end(); cit2++)
if (cit2->second == c) {
printf(" %s\n",
cit2->first->label().c_str());
if (outFile) {
t = new NTriangulation(*(cit2->first));
t->setLabel(cit2->first->label());
classCnt->insertChildLast(t);
}
cit2->second = -1;
}
printf("\n");
}
}
printf("Final statistics:\n");
printf(" Triangulations read: %ld\n", nTris);
printf(" Equivalence classes: %ld\n", nClasses);
printf(" Non-minimal triangulations: %ld\n", nNonMin);
printf(" Triangulations with new equivs: %ld\n", nHasNew);
// Are we saving results?
if (outFile && newTree) {
ctrlLogStamp() << "Saving results to " << outFile << "." << std::endl;
newTree->save(outFile);
} else
ctrlLogStamp() << "Not saving results." << std::endl;
// Clean up and exit.
if (newTree)
delete newTree;
if (hasError) {
ctrlLogStamp() << "ERROR: One or more errors occurred; "
"read back through the log for details." << std::endl;
printf("\nERROR: One or more errors occurred.\n");
printf( " Please read through the log file %s for details.\n",
logFile);
} else
ctrlLogStamp() << "All done." << std::endl;
return 0;
}
/**
* Controller helper routine.
*
* Wait for the next running slave to finish a task. Note that if no
* slaves are currently processing tasks, this routine will block forever!
*/
int ctrlWaitForSlave() {
long result;
MPI_Status status;
MPI_Recv(&result, 1, MPI_LONG, MPI_ANY_SOURCE, TAG_RESULT, MPI_COMM_WORLD,
&status);
nRunningSlaves--;
int slave = status.MPI_SOURCE;
ctrlLogStamp() << "Task completed by slave " << slave << "." << std::endl;
char triLabel[MAX_TRI_LABEL_LEN + 1];
if (result == RESULT_OK || result == RESULT_HAS_NEW) {
if (result == RESULT_HAS_NEW) {
// The original packet label comes through first so we can
// write it to the log. It will come through again shortly
// as part of the set of equivalent triangulations.
MPI_Recv(triLabel, MAX_TRI_LABEL_LEN + 1, MPI_CHAR, slave,
TAG_RESULT_DATA, MPI_COMM_WORLD, &status);
ctrlLogStamp() << "WARNING: Has unseen equivalent: " << triLabel
<< std::endl;
nHasNew++;
}
equivs.clear();
NPacket* p;
NTriangulation* tri;
while (1) {
MPI_Recv(triLabel, MAX_TRI_LABEL_LEN + 1, MPI_CHAR, slave,
TAG_RESULT_DATA, MPI_COMM_WORLD, &status);
if (*triLabel == 0)
break;
p = tree->findPacketLabel(triLabel);
if (! p) {
ctrlLogStamp() << "ERROR: Returned equivalent [" << triLabel
<< "] not found." << std::endl;
hasError = true;
} else {
tri = dynamic_cast<NTriangulation*>(p);
if (! tri) {
ctrlLogStamp() << "ERROR: Returned equivalent [" << triLabel
<< "] is not a triangulation!" << std::endl;
hasError = true;
} else
equivs.insert(tri);
}
}
ctrlLogStamp() << "Resulting set contains "
<< equivs.size() << " equivalent(s)." << std::endl;
// In equivs we now have a list of all triangulations
// equivalent to orig.
// Is this an equivalence class we're already seen?
TriSet::iterator tit;
ClassMap::iterator cit, cit2;
for (tit = equivs.begin(); tit != equivs.end(); tit++) {
cit = eClass.find(*tit);
if (cit != eClass.end())
break;
}
if (tit != equivs.end()) {
// We found an equivalence class. Insert everything we
// haven't seen yet, and merge the classes of everything
// we have.
int c, cOld;
c = cit->second;
for (tit = equivs.begin(); tit != equivs.end(); tit++) {
cit = eClass.find(*tit);
if (cit == eClass.end())
eClass.insert(std::make_pair(*tit, c));
else if (cit->second != c) {
// Merge the two equivalence classes.
cOld = cit->second;
for (cit = eClass.begin(); cit != eClass.end(); cit++)
if (cit->second == cOld)
cit->second = c;
nClasses--;
}
}
} else {
// No such equivalence class. Insert everything.
int c = nextClass++;
for (tit = equivs.begin(); tit != equivs.end(); tit++)
eClass.insert(std::make_pair(*tit, c));
nClasses++;
}
} else if (result == RESULT_NON_MINIMAL) {
MPI_Recv(triLabel, MAX_TRI_LABEL_LEN + 1, MPI_CHAR, slave,
TAG_RESULT_DATA, MPI_COMM_WORLD, &status);
ctrlLogStamp() << "Non-minimal triangulation: " << triLabel
<< std::endl;
nNonMin++;
} else if (result == RESULT_ERR) {
char errMsg[MAX_ERR_MSG_LEN + 1];
MPI_Recv(errMsg, MAX_ERR_MSG_LEN + 1, MPI_CHAR, slave,
TAG_RESULT_DATA, MPI_COMM_WORLD, &status);
ctrlLogStamp() << "ERROR: " << errMsg << std::endl;
hasError = true;
} else {
ctrlLogStamp() << "ERROR: Unknown result code " << result
<< " received from slave." << std::endl;
hasError = true;
}
return slave;
}
int main (int argc, char *argv[]) {
// handle cmd args
int batch_size, maxiter;
std::string datadir;
std::string output_file;
if ( argc > 5 || argc < 2 ) {
printf( "Usage: ./logistic_mpi <data_directory> <batch_size> "
"<max_iterations> <model_output_file>\n");
MPI_Finalize();
exit( 0 );
} else if ( argc == 5 ) {
datadir = argv[1];
batch_size = atoi( argv[2] ); // mini-batch processing
if ( batch_size == -1 ) { batch_size = INT_MIN; }
maxiter = atoi( argv[3] );
output_file = argv[4];
} else if ( argc == 4 ) {
datadir = argv[1];
batch_size = atoi( argv[2] ); // mini-batch processing
if ( batch_size == -1 ) { batch_size = INT_MIN; }
maxiter = atoi( argv[3] );
output_file = "logistic.model";
} else if ( argc == 4 ) {
datadir = argv[1];
batch_size = atoi( argv[2] ); // mini-batch processing
if ( batch_size == -1 ) { batch_size = INT_MIN; }
maxiter = 100;
output_file = "logistic.model";
} else {
datadir = argv[1];
batch_size = INT_MIN; // batch processing
maxiter = 100;
output_file = "logistic.model";
}
// initialize/populate mpi specific vars local to each node
double t1,t2; // elapsed time computation
int numtasks, taskid, len;
char hostname[MPI_MAX_PROCESSOR_NAME];
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &numtasks);
MPI_Comm_rank(MPI_COMM_WORLD,&taskid);
MPI_Get_processor_name(hostname, &len);
MPI_Op op;
/* DATA PREPROCESSING */
if ( taskid == MASTER ) {
printf( "\nLoading and Preprocessing Data\n" );
}
t1 = MPI_Wtime();
// determine number of instances
DataVec datavec;
mlu::count_instances( datadir, datavec, num_inst );
// determine number of features
mlu::count_features( datavec[0], n );
/* DATA INITIALIZATION */
// randomize instances
std::random_shuffle( datavec.begin(), datavec.end() );
// partition data based on taskid
size_t div = datavec.size() / numtasks;
ProbSize limit = ( taskid == numtasks - 1 ) ? num_inst : div * ( taskid + 1 );
m = limit - div * taskid;
// danamically allocate data
Mat X( m, n );
Vec labels( m );
// load data partition
double feat_val, label;
ProbSize i = 0;
for ( ProbSize idx = taskid * div; idx < limit; ++idx ) {
std::ifstream data( datavec[idx] );
for ( ProbSize j=0; j<n; ++j ) {
data >> feat_val;
X(i,j) = feat_val;
}
data >> label;
labels[i] = label;
i++;
}
// perform feature scaling (optional)
if ( scaling ) {
// Allreduce to find global min
Vec X_min_tmp = X.colwise().minCoeff();
X_min_data = X_min_tmp.data();
Vec X_min = Vec( X_min_tmp.size() );
MPI_Allreduce( X_min_tmp.data(), X_min.data(), X_min_tmp.size(), MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD );
// Allreduce to find global max
Vec X_max_tmp = X.colwise().maxCoeff();
X_max_data = X_max_tmp.data();
Vec X_max = Vec( X_max_tmp.size() );
MPI_Allreduce( X_max_tmp.data(), X_max.data(), X_max_tmp.size(), MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD );
// scale features using global min and max
mlu::scale_features( X, X_min, X_max, 1, 0 );
}
/* FORMAT LABELS */
// get unique labels
mlu::get_unique_labels( labels, classmap );
// allreduce to obtain maximum label set size
int local_size = classmap.size();
int max_size = 0;
MPI_Allreduce( &local_size, &max_size, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
// allreduce to obtain global unique label set
int unique_labels[max_size];
int global_unique_labels[max_size];
for ( int i=0; i<max_size; ++i ) {
unique_labels[i] = -1;
global_unique_labels[i] = -1;
}
int idx = 0;
for ( auto& kv : classmap ) {
unique_labels[idx++] = kv.first;
}
MPI_Op_create( (MPI_User_function *)reduce_unique_labels, 1, &op );
MPI_Allreduce( unique_labels, global_unique_labels, max_size, MPI_INT, op, MPI_COMM_WORLD );
MPI_Op_free( &op );
// update local classmap
std::sort( global_unique_labels, global_unique_labels + max_size );
classmap.clear();
int labeltmp;
idx=0;
for ( int i=0; i<max_size; ++i ) {
labeltmp = global_unique_labels[i];
if ( labeltmp != -1 ) {
classmap.emplace( labeltmp, idx++ );
}
}
// format the local label set into a matrix based on global class map
Mat y = mlu::format_labels( labels, classmap );
numlabels = (LayerSize) classmap.size();
// output total data loading time for each task
MPI_Barrier( MPI_COMM_WORLD );
t2 = MPI_Wtime();
printf( "--- task %d loading time %lf\n", taskid, t2 - t1 );
/* INIT LOCAL CLASSIFIER */
LogisticRegression logistic_layer( n, numlabels, true );
/* OPTIMIZATION */
if ( taskid == MASTER ) {
printf( "\nPerforming Gradient Descent\n" );
}
int update_size; // stores the number of instances read for each update
double grad_mag; // stores the magnitude of the gradient for each update
int delta_size = logistic_layer.get_theta_size();
Vec delta_update = Vec::Zero( delta_size );
int global_update_size;
if ( taskid == MASTER ) {
printf( "iteration : elapsed time : magnitude\n" );
}
for ( int i=0; i<maxiter; ++i ) {
// compute gradient update
t1 = MPI_Wtime();
logistic_layer.compute_gradient( X, y, batch_size, update_size );
delta_data = logistic_layer.get_delta().data();
// sum updates across all partitions
MPI_Allreduce(
delta_data,
delta_update.data(),
delta_size,
MPI_DOUBLE,
MPI_SUM,
MPI_COMM_WORLD
);
logistic_layer.set_delta( delta_update );
// sum the update sizes
MPI_Allreduce( &update_size, &global_update_size, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
// normalize + regularize gradient update
logistic_layer.normalize_gradient( global_update_size );
logistic_layer.regularize_gradient( global_update_size );
// update logistic_layer parameters
t2 = MPI_Wtime();
if ( logistic_layer.converged( grad_mag ) ) { break; }
if ( taskid == MASTER ) {
printf( "%d : %lf : %lf\n", i+1, t2 - t1, grad_mag );
}
logistic_layer.update_theta();
}
/* MODEL STORAGE */
if (taskid == MASTER) {
FILE *output;
output = fopen ( output_file.c_str(), "w" );
int idx;
Vec theta = logistic_layer.get_theta();
printf( "\nWriting Model to File: %s\n\n", output_file.c_str() );
fprintf( output, "%lu\n", theta.size() );
for ( idx=0; idx<theta.size()-1; ++idx ) {
fprintf( output, "%lf\t", theta[idx] );
}
fprintf( output, "%lf\n", theta[idx] );
fclose( output );
}
MPI_Finalize();
return 0;
}
void processTree() {
TriSet::iterator tit;
ClassMap::iterator cit, cit2;
int c, cOld;
NTriangulation* t;
for (NPacket* p = tree; p; p = p->nextTreePacket())
if (p->type() == PACKET_TRIANGULATION) {
// A triangulation to process.
t = static_cast<NTriangulation*>(p);
fprintf(stderr, "Processing %s ...\n", t->label().c_str());
nTris++;
nonMin = false;
orig = static_cast<NTriangulation*>(p);
equivs.clear();
equivs.insert(orig);
tryMovesUp(orig, argUp);
if (nonMin) {
allNonMin.push_back(orig);
nNonMin++;
continue;
}
// In equivs we now have a list of all triangulations
// equivalent to orig.
// Is this an equivalence class we're already seen?
for (tit = equivs.begin(); tit != equivs.end(); tit++) {
cit = eClass.find(*tit);
if (cit != eClass.end())
break;
}
if (tit != equivs.end()) {
// We found an equivalence class. Insert everything we
// haven't seen yet, and merge the classes of everything
// we have.
c = cit->second;
for (tit = equivs.begin(); tit != equivs.end(); tit++) {
cit = eClass.find(*tit);
if (cit == eClass.end())
eClass.insert(std::make_pair(*tit, c));
else if (cit->second != c) {
// Merge the two equivalence classes.
cOld = cit->second;
for (cit = eClass.begin(); cit != eClass.end(); cit++)
if (cit->second == cOld)
cit->second = c;
nClasses--;
}
}
} else {
// No such equivalence class. Insert everything.
c = nextClass++;
for (tit = equivs.begin(); tit != equivs.end(); tit++)
eClass.insert(std::make_pair(*tit, c));
nClasses++;
}
}
// Finished progress reporting.
fprintf(stderr, "\n");
// Write the summary of results.
if (! allNonMin.empty()) {
printf("NON-MINIMAL TRIANGULATIONS:\n\n");
for (std::list<NTriangulation*>::const_iterator it = allNonMin.begin();
it != allNonMin.end(); it++)
printf(" %s\n", (*it)->label().c_str());
printf("\n");
}
if (nClasses) {
printf("EQUIVALENCE CLASSES:\n\n");
if (outFile) {
newTree = new NContainer();
newTree->setLabel("Equivalence Classes");
}
int classNum = 1;
std::string className;
NContainer* classCnt = 0;
for (cit = eClass.begin(); cit != eClass.end(); cit++)
if (cit->second >= 0) {
// The first triangulation of a new equivalence class.
c = cit->second;
std::ostringstream s;
s << "Class " << classNum << " : " <<
cit->first->homology().str();
className = s.str();
classNum++;
printf("%s\n\n", className.c_str());
if (outFile) {
classCnt = new NContainer();
classCnt->setLabel(className);
newTree->insertChildLast(classCnt);
}
// Find the triangulations in this class, and erase the
// class as we go.
for (cit2 = cit; cit2 != eClass.end(); cit2++)
if (cit2->second == c) {
printf(" %s\n",
cit2->first->label().c_str());
if (outFile) {
t = new NTriangulation(*(cit2->first));
t->setLabel(cit2->first->label());
classCnt->insertChildLast(t);
}
cit2->second = -1;
}
printf("\n");
}
}
printf("Final statistics:\n");
printf(" Triangulations read: %ld\n", nTris);
printf(" Equivalence classes: %ld\n", nClasses);
printf(" New triangulations: %ld\n", nNew);
printf(" Non-minimal triangulations: %ld\n", nNonMin);
}
