void* calcConProbs(void* arg) {
Params *params = (Params*)arg;
ModelType *model = (ModelType*)(params->model);
ReadReader<ReadType> *reader = (ReadReader<ReadType>*)(params->reader);
HitContainer<HitType> *hitv = (HitContainer<HitType>*)(params->hitv);
double *ncpv = (double*)(params->ncpv);
ReadType read;
READ_INT_TYPE N = hitv->getN();
HIT_INT_TYPE fr, to;
assert(model->getNeedCalcConPrb());
reader->reset();
for (READ_INT_TYPE i = 0; i < N; i++) {
general_assert(reader->next(read), "Can not load a read!");
fr = hitv->getSAt(i);
to = hitv->getSAt(i + 1);
ncpv[i] = model->getNoiseConPrb(read);
for (HIT_INT_TYPE j = fr; j < to; j++) {
HitType &hit = hitv->getHitAt(j);
hit.setConPrb(model->getConPrb(read, hit));
}
}
return NULL;
}
void processquerypattern(ModelType & model, ClassDecoder * classdecoder, const Pattern & pattern, const string & dorelations, bool doinstantiate) {
if (!model.has(pattern)) {
cout << "PATTERN \"" << pattern.tostring(*classdecoder) << "\" NOT FOUND IN MODEL" << endl;
} else {
model.print(&cout, *classdecoder, pattern, doinstantiate);
if (!dorelations.empty()) model.outputrelations(pattern, *classdecoder, &cout, dorelations == "all" ? "" : dorelations);
}
}
void* GET_SS_STEP(void* arg) {
Params *params = (Params*)arg;
ModelType *model = (ModelType*)(params->model);
ReadReader<ReadType> *reader = (ReadReader<ReadType>*)(params->reader);
HitContainer<HitType> *hitv = (HitContainer<HitType>*)(params->hitv);
double *ncpv = (double*)(params->ncpv);
ModelType *mhp = (ModelType*)(params->mhp);
assert(!model->getNeedCalcConPrb());
ReadType read;
READ_INT_TYPE N = hitv->getN();
double sum;
vector<double> fracs; //to remove this, do calculation twice
HIT_INT_TYPE fr, to, id;
reader->reset();
mhp->init();
for (READ_INT_TYPE i = 0; i < N; i++) {
general_assert(reader->next(read), "Can not load a read!");
fr = hitv->getSAt(i);
to = hitv->getSAt(i + 1);
fracs.resize(to - fr + 1);
sum = 0.0;
fracs[0] = probv[0] * ncpv[i];
if (fracs[0] < EPSILON) fracs[0] = 0.0;
sum += fracs[0];
for (HIT_INT_TYPE j = fr; j < to; j++) {
HitType &hit = hitv->getHitAt(j);
id = j - fr + 1;
fracs[id] = probv[hit.getSid()] * hit.getConPrb();
if (fracs[id] < EPSILON) fracs[id] = 0.0;
sum += fracs[id];
}
if (sum >= EPSILON) {
fracs[0] /= sum;
//mhp->updateNoise(read, fracs[0]);
for (HIT_INT_TYPE j = fr; j < to; j++) {
HitType &hit = hitv->getHitAt(j);
id = j - fr + 1;
fracs[id] /= sum;
mhp->update(read, hit, fracs[id]);
}
}
}
return NULL;
}
SizeType dofsFromModelName(const ModelType& model)
{
const PsimagLite::String& modelName = model.params().model;
SizeType site = 0; // FIXME : account for Hilbert spaces changing with site
SizeType dofs = SizeType(log(model.hilbertSize(site))/log(2.0));
std::cerr<<"DOFS= "<<dofs<<" <------------------------------------\n";
if (modelName.find("FeAsBasedSc")!=PsimagLite::String::npos) return dofs;
if (modelName.find("FeAsBasedScExtended")!=PsimagLite::String::npos) return dofs;
if (modelName.find("HubbardOneBand")!=PsimagLite::String::npos) return dofs;
// max number here, site dependence taken into account elsewhere
if (modelName.find("Immm")!=PsimagLite::String::npos) return 4;
return 0;
}
//Classification
void CARTTrainer::train(ModelType& model, ClassificationDataset const& dataset){
//Store the number of input dimensions
m_inputDimension = inputDimension(dataset);
//Pass input dimension (i.e., number of attributes) to tree model
model.setInputDimension(m_inputDimension);
//Find the largest label, so we know how big the histogram should be
m_maxLabel = static_cast<unsigned int>(numberOfClasses(dataset))-1;
// create cross-validation folds
ClassificationDataset set=dataset;
CVFolds<ClassificationDataset> folds = createCVSameSizeBalanced(set, m_numberOfFolds);
//find the best tree for the cv folds
double bestErrorRate = std::numeric_limits<double>::max();
CARTClassifier<RealVector>::TreeType bestTree;
//Run through all the cross validation sets
for (unsigned fold = 0; fold < m_numberOfFolds; ++fold) {
ClassificationDataset dataTrain = folds.training(fold);
ClassificationDataset dataTest = folds.validation(fold);
//Create attribute tables
//O.K. stores how often label(i) can be found in the dataset
//O.K. TODO: std::vector<unsigned int> is sufficient
boost::unordered_map<std::size_t, std::size_t> cAbove = createCountMatrix(dataTrain);
AttributeTables tables = createAttributeTables(dataTrain.inputs());
//create initial tree for the fold
CARTClassifier<RealVector>::TreeType tree = buildTree(tables, dataTrain, cAbove, 0);
model.setTree(tree);
while(true){
ZeroOneLoss<unsigned int, RealVector> loss;
double errorRate = loss.eval(dataTest.labels(), model(dataTest.inputs()));
if(errorRate < bestErrorRate){
//We have found a subtree that has a smaller error rate when tested!
bestErrorRate = errorRate;
bestTree = tree;
}
if(tree.size()!=1) break;
pruneTree(tree);
model.setTree(tree);
}
}
SHARK_CHECK(bestTree.size() > 0, "We should never set a tree that is empty.");
model.setTree(bestTree);
}
void sample_theta_vectors_from_count_vectors() {
ModelType model;
model.read(modelF);
calcExpectedEffectiveLengths<ModelType>(model);
int num_threads = min(nThreads, nCV);
buffer = new Buffer(nMB, nSamples, cvlen, tmpF);
paramsArray = new Params[num_threads];
threads = new pthread_t[num_threads];
char inpF[STRLEN];
for (int i = 0; i < num_threads; i++) {
paramsArray[i].no = i;
sprintf(inpF, "%s%d", cvsF, i);
paramsArray[i].fi = fopen(inpF, "r");
paramsArray[i].engine = engineFactory::new_engine();
paramsArray[i].mw = model.getMW();
}
/* set thread attribute to be joinable */
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
for (int i = 0; i < num_threads; i++) {
rc = pthread_create(&threads[i], &attr, &sample_theta_from_c, (void*)(¶msArray[i]));
pthread_assert(rc, "pthread_create", "Cannot create thread " + itos(i) + " (numbered from 0) in sample_theta_vectors_from_count_vectors!");
}
for (int i = 0; i < num_threads; i++) {
rc = pthread_join(threads[i], &status);
pthread_assert(rc, "pthread_join", "Cannot join thread " + itos(i) + " (numbered from 0) in sample_theta_vectors_from_count_vectors!");
}
/* destroy attribute */
pthread_attr_destroy(&attr);
delete[] threads;
for (int i = 0; i < num_threads; i++) {
fclose(paramsArray[i].fi);
delete paramsArray[i].engine;
}
delete[] paramsArray;
delete buffer; // Must delete here, force the content left in the buffer be written into the disk
if (verbose) { printf("Sampling is finished!\n"); }
}
//Train model with a regression dataset
void CARTTrainer::train(ModelType& model, RegressionDataset const& dataset)
{
//Store the number of input dimensions
m_inputDimension = inputDimension(dataset);
//Pass input dimension (i.e., number of attributes) to tree model
model.setInputDimension(m_inputDimension);
//Store the size of the labels
m_labelDimension = labelDimension(dataset);
// create cross-validation folds
RegressionDataset set=dataset;
CVFolds<RegressionDataset > folds = createCVSameSize(set, m_numberOfFolds);
double bestErrorRate = std::numeric_limits<double>::max();
CARTClassifier<RealVector>::TreeType bestTree;
for (unsigned fold = 0; fold < m_numberOfFolds; ++fold){
//Run through all the cross validation sets
RegressionDataset dataTrain = folds.training(fold);
RegressionDataset dataTest = folds.validation(fold);
std::size_t numTrainElements = dataTrain.numberOfElements();
AttributeTables tables = createAttributeTables(dataTrain.inputs());
std::vector < RealVector > labels(numTrainElements);
boost::copy(dataTrain.labels().elements(),labels.begin());
//Build tree form this fold
CARTClassifier<RealVector>::TreeType tree = buildTree(tables, dataTrain, labels, 0, dataTrain.numberOfElements());
//Add the tree to the model and prune
model.setTree(tree);
while(true){
//evaluate the error of current tree
SquaredLoss<> loss;
double error = loss.eval(dataTest.labels(), model(dataTest.inputs()));
if(error < bestErrorRate){
//We have found a subtree that has a smaller error rate when tested!
bestErrorRate = error;
bestTree = tree;
}
if(tree.size() == 1) break;
pruneTree(tree);
model.setTree(tree);
}
}
SHARK_CHECK(bestTree.size() > 0, "We should never set a tree that is empty.");
model.setTree(bestTree);
}
agent_framework::model::Metric get_metric(const ModelType<TYPE>& resource, const ResourceSensorPtr resource_sensor) {
const auto metrics = agent_framework::module::get_manager<agent_framework::model::Metric>().get_entries(
[&resource, &resource_sensor](const agent_framework::model::Metric& metric) -> bool {
return metric.get_component_type() == resource.get_component()
&& metric.get_component_uuid() == resource.get_uuid()
&& metric.get_metric_definition_uuid() == resource_sensor->get_definition().get_uuid();
}
);
if (metrics.size() != 1) {
throw std::runtime_error("Invalid number of Metrics (" + std::to_string(metrics.size()) + ") for "
+ resource_sensor->get_definition().get_metric_jsonptr() + " reading for "
+ resource.get_component().to_string() + " with uuid " + resource.get_uuid() );
}
return metrics.front();
}
void calcExpectedEffectiveLengths(ModelType& model) {
int lb, ub, span;
double *pdf = NULL, *cdf = NULL, *clen = NULL; // clen[i] = sigma_{j=1}^{i}pdf[i]*(lb+i)
model.getGLD().copyTo(pdf, cdf, lb, ub, span);
clen = new double[span + 1];
clen[0] = 0.0;
for (int i = 1; i <= span; i++) {
clen[i] = clen[i - 1] + pdf[i] * (lb + i);
}
eel.clear();
eel.resize(M + 1, 0.0);
for (int i = 1; i <= M; i++) {
int totLen = refs.getRef(i).getTotLen();
int fullLen = refs.getRef(i).getFullLen();
int pos1 = max(min(totLen - fullLen + 1, ub) - lb, 0);
int pos2 = max(min(totLen, ub) - lb, 0);
if (pos2 == 0) { eel[i] = 0.0; continue; }
eel[i] = fullLen * cdf[pos1] + ((cdf[pos2] - cdf[pos1]) * (totLen + 1) - (clen[pos2] - clen[pos1]));
assert(eel[i] >= 0);
if (eel[i] < MINEEL) { eel[i] = 0.0; }
}
delete[] pdf;
delete[] cdf;
delete[] clen;
}
void verifyPlaneSac (ModelType & model, SacType & sac, unsigned int inlier_number = 2000, float tol = 1e-1f,
float refined_tol = 1e-1f, float proj_tol = 1e-3f)
{
// Algorithm tests
bool result = sac.computeModel ();
ASSERT_EQ (result, true);
std::vector<int> sample;
sac.getModel (sample);
EXPECT_EQ (int (sample.size ()), 3);
std::vector<int> inliers;
sac.getInliers (inliers);
EXPECT_GE (int (inliers.size ()), inlier_number);
Eigen::VectorXf coeff;
sac.getModelCoefficients (coeff);
EXPECT_EQ (int (coeff.size ()), 4);
EXPECT_NEAR (coeff[0]/coeff[3], plane_coeffs_[0], tol);
EXPECT_NEAR (coeff[1]/coeff[3], plane_coeffs_[1], tol);
EXPECT_NEAR (coeff[2]/coeff[3], plane_coeffs_[2], tol);
Eigen::VectorXf coeff_refined;
model->optimizeModelCoefficients (inliers, coeff, coeff_refined);
EXPECT_EQ (int (coeff_refined.size ()), 4);
EXPECT_NEAR (coeff_refined[0]/coeff_refined[3], plane_coeffs_[0], refined_tol);
EXPECT_NEAR (coeff_refined[1]/coeff_refined[3], plane_coeffs_[1], refined_tol);
// This test fails in Windows (VS 2010) -- not sure why yet -- relaxing the constraint from 1e-2 to 1e-1
// This test fails in MacOS too -- not sure why yet -- disabling
//EXPECT_NEAR (coeff_refined[2]/coeff_refined[3], plane_coeffs_[2], refined_tol);
// Projection tests
PointCloud<PointXYZ> proj_points;
model->projectPoints (inliers, coeff_refined, proj_points);
EXPECT_NEAR (proj_points.points[20].x, 1.1266, proj_tol);
EXPECT_NEAR (proj_points.points[20].y, 0.0152, proj_tol);
EXPECT_NEAR (proj_points.points[20].z, -0.0156, proj_tol);
EXPECT_NEAR (proj_points.points[30].x, 1.1843, proj_tol);
EXPECT_NEAR (proj_points.points[30].y, -0.0635, proj_tol);
EXPECT_NEAR (proj_points.points[30].z, -0.0201, proj_tol);
EXPECT_NEAR (proj_points.points[50].x, 1.0749, proj_tol);
EXPECT_NEAR (proj_points.points[50].y, -0.0586, proj_tol);
EXPECT_NEAR (proj_points.points[50].z, 0.0587, refined_tol);
}
void viewmodel(ModelType & model, ClassDecoder * classdecoder, ClassEncoder * classencoder, bool print, bool report, bool nocoverage, bool histogram , bool query, const string dorelations, bool doinstantiate, bool info, bool printreverseindex, int cooc, double coocthreshold = 0.1) {
cerr << "Generating desired views..." << endl;
if (print) {
if (classdecoder == NULL) {
cerr << "ERROR: Unable to print model, no class file specified (--classfile)" << endl;
} else {
model.print(&cout, *classdecoder, doinstantiate);
}
}
if (printreverseindex) {
model.printreverseindex(&cout, *classdecoder);
}
if (report) {
model.report(&cout, nocoverage);
}
if (histogram) {
model.histogram(&cout);
}
if (cooc == 2) {
model.outputcooc_npmi(&cout, *classdecoder,coocthreshold);
} else if (cooc == 1) {
model.outputcooc(&cout, *classdecoder,coocthreshold);
}
if (query) {
if (classencoder == NULL) {
cerr << "ERROR: Unable to query model, no class encoder specified (--classfile)" << endl;
} else {
querymodel<ModelType>(model, classencoder, classdecoder, dorelations, doinstantiate);
}
} else if (!dorelations.empty()) {
bool first = true;
for (typename ModelType::iterator iter = model.begin(); iter != model.end(); iter++) {
cout << iter->first.tostring(*classdecoder) << endl;
const PatternPointer pp = iter->first;
model.outputrelations(pp, *classdecoder, &cout, dorelations == "all" ? "" : dorelations,first);
first = false;
}
}
if (info) {
model.info(&cout);
}
}
void querymodel(ModelType & model, ClassEncoder * classencoder, ClassDecoder * classdecoder, string dorelations, bool doinstantiate, bool repeat = true) {
const bool allowunknown = true;
unsigned char buffer[65536];
uint32_t linenum = 0;
std::string line;
cerr << "Colibri Patternmodeller -- Interactive query mode." << endl;
cerr << " Type ctrl-D to quit, type X to switch between exact mode and extensive mode (default: extensive mode)." << endl;
bool exact = false;
do {
linenum++;
cerr << linenum << ">> ";
getline(cin,line);
if ((line == "X") || (line == "X\n")) {
exact = !exact;
if (exact) {
cerr << "Switched to Exact mode - Only exact matches will be shown now" << endl;
} else {
cerr << "Switched to Extensive mode - Input will be scanned for all matching patterns" << endl;
}
} else if (!line.empty()) {
const int buffersize = classencoder->encodestring(line, buffer, allowunknown);
Pattern linepattern = Pattern(buffer, buffersize);
if (exact) {
processquerypattern<ModelType>(model,classdecoder, linepattern, dorelations, doinstantiate);
} else {
vector<pair<Pattern, int> > patterns = model.getpatterns(linepattern);
if (model.has(linepattern)) {
const IndexReference ref = IndexReference(linenum,0);
//process and output instance
cout << ref.sentence << ':' << (int) ref.token << "\t";
processquerypattern<ModelType>(model, classdecoder, linepattern, dorelations, doinstantiate);
}
for (vector<pair<Pattern,int> >::iterator iter = patterns.begin(); iter != patterns.end(); iter++) {
const Pattern pattern = iter->first;
const IndexReference ref = IndexReference(linenum,iter->second);
//process and output instance
cout << ref.sentence << ':' << (int) ref.token << "\t";
processquerypattern<ModelType>(model, classdecoder, pattern, dorelations, doinstantiate);
}
}
}
} while (!cin.eof() && (repeat));
}
void NormalizeComponentsWhitening::train(ModelType& model, UnlabeledData<RealVector> const& input){
std::size_t dc = dataDimension(input);
SHARK_CHECK(input.numberOfElements() >= dc + 1, "[NormalizeComponentsWhitening::train] input needs to contain more points than there are input dimensions");
SHARK_CHECK(m_targetVariance > 0.0, "[NormalizeComponentsWhitening::train] target variance must be positive");
// dense model with bias having input and output dimension equal to data dimension
model.setStructure(dc, dc, true);
RealVector mean;
RealMatrix covariance;
meanvar(input, mean, covariance);
RealMatrix whiteningMatrix = createWhiteningMatrix(covariance);
whiteningMatrix *= std::sqrt(m_targetVariance);
RealVector offset = -prod(trans(whiteningMatrix),mean);
model.setStructure(RealMatrix(trans(whiteningMatrix)), offset);
}
void Painter::Render(D3D* d3d, Frustum* frustum, Viewport* viewport,
D3DXMATRIX world, D3DXMATRIX view, D3DXMATRIX projection, D3DXMATRIX ortho)
{
std::sort(z.begin(), z.end(), Compare);
for( vector<BaseType*>::iterator p=z.begin(); p!=z.end(); ++p)
{
//process(*p); RENDER BACK TO FRONT
//FRONT = negatives
//BACK = positives
//sort list greatest to least so back renders first
BaseType* base = *p;
int type = base->GetType();
if (type == 1)
{
ModelType* model = (ModelType*)base;
model->transform = &xforce(*model->transform, Rad);
model->Render(d3d->GetDeviceContext(), frustum, viewport);
}
if (type == 2)
{
TextType* text = (TextType*)base;
text->Render(d3d->GetDeviceContext(), viewport);
}
if (type == 3)
{
BitmapType* bitmap = (BitmapType*)base;
bitmap->transform = &xforce(*bitmap->transform, Rad);
bitmap->Render(d3d, viewport);
}
/*if (type == 4)
{
TransformType* transform = (TransformType*)base;
transform->Render(d3d, world, view, projection, ortho);
}*/
}
return;
}
ClassNodeDrawOptions ClassGraph::getComponentOptions(ModelType const &type,
ClassNodeDrawOptions const &options)
{
ClassNodeDrawOptions compOptions = options;
bool mainComponent = false;
if(mNodes.size() > FIRST_CLASS_INDEX)
{
ModelType const *mainType = mNodes[FIRST_CLASS_INDEX].getType();
ModelClassifier const *mainClass = mainType->getClass();
ModelClassifier const *testClass = type.getClass();
if(mainClass && testClass)
{
ModelModule const *mainModule = mainClass->getModule();
ModelModule const *testModule = testClass->getModule();
if(mainModule && testModule)
{
FilePath mainFp(mainModule->getName(), FP_File);
FilePath fp(testClass->getModule()->getName(), FP_File);
if(mainFp.getDrivePath() == fp.getDrivePath())
{
mainComponent = true;
}
}
}
}
else
{
mainComponent = true;
}
if(!mainComponent)
{
compOptions.drawAttrTypes = false;
compOptions.drawAttributes = false;
compOptions.drawOperParams = false;
compOptions.drawOperations = false;
compOptions.drawPackageName = true;
}
return compOptions;
}
bool QueryResponse::operator==(const ModelType &rhs) const {
return queryHash() == rhs.queryHash() and get() == rhs.get();
}
bool AssetResponse::operator==(const ModelType &rhs) const {
return asset() == rhs.asset();
}
void writeResults(ModelType& model, double* counts) {
double denom;
char outF[STRLEN];
FILE *fo;
sprintf(modelF, "%s.model", statName);
model.write(modelF);
//calculate tau values
double *tau = new double[M + 1];
memset(tau, 0, sizeof(double) * (M + 1));
denom = 0.0;
for (int i = 1; i <= M; i++)
if (eel[i] >= EPSILON) {
tau[i] = theta[i] / eel[i];
denom += tau[i];
}
general_assert(denom > 0, "No alignable reads?!");
for (int i = 1; i <= M; i++) {
tau[i] /= denom;
}
//isoform level results
sprintf(outF, "%s.iso_res", imdName);
fo = fopen(outF, "w");
for (int i = 1; i <= M; i++) {
const Transcript& transcript = transcripts.getTranscriptAt(i);
fprintf(fo, "%s%c", transcript.getTranscriptID().c_str(), (i < M ? '\t' : '\n'));
}
for (int i = 1; i <= M; i++)
fprintf(fo, "%.2f%c", counts[i], (i < M ? '\t' : '\n'));
for (int i = 1; i <= M; i++)
fprintf(fo, "%.15g%c", tau[i], (i < M ? '\t' : '\n'));
for (int i = 1; i <= M; i++) {
const Transcript& transcript = transcripts.getTranscriptAt(i);
fprintf(fo, "%s%c", transcript.getGeneID().c_str(), (i < M ? '\t' : '\n'));
}
fclose(fo);
//gene level results
sprintf(outF, "%s.gene_res", imdName);
fo = fopen(outF, "w");
for (int i = 0; i < m; i++) {
const string& gene_id = transcripts.getTranscriptAt(gi.spAt(i)).getGeneID();
fprintf(fo, "%s%c", gene_id.c_str(), (i < m - 1 ? '\t' : '\n'));
}
for (int i = 0; i < m; i++) {
double sumC = 0.0; // sum of counts
int b = gi.spAt(i), e = gi.spAt(i + 1);
for (int j = b; j < e; j++) sumC += counts[j];
fprintf(fo, "%.2f%c", sumC, (i < m - 1 ? '\t' : '\n'));
}
for (int i = 0; i < m; i++) {
double sumT = 0.0; // sum of tau values
int b = gi.spAt(i), e = gi.spAt(i + 1);
for (int j = b; j < e; j++) sumT += tau[j];
fprintf(fo, "%.15g%c", sumT, (i < m - 1 ? '\t' : '\n'));
}
for (int i = 0; i < m; i++) {
int b = gi.spAt(i), e = gi.spAt(i + 1);
for (int j = b; j < e; j++) {
fprintf(fo, "%s%c", transcripts.getTranscriptAt(j).getTranscriptID().c_str(), (j < e - 1 ? ',' : (i < m - 1 ? '\t' :'\n')));
}
}
fclose(fo);
delete[] tau;
if (verbose) { printf("Expression Results are written!\n"); }
}
void writeResults(ModelType& model, double* counts) {
sprintf(modelF, "%s.model", statName);
model.write(modelF);
writeResultsEM(M, refName, imdName, transcripts, theta, eel, countvs[0]);
}
bool Query::operator==(const ModelType &rhs) const {
return this->get() == rhs.get();
}
void* E_STEP(void* arg) {
Params *params = (Params*)arg;
ModelType *model = (ModelType*)(params->model);
ReadReader<ReadType> *reader = (ReadReader<ReadType>*)(params->reader);
HitContainer<HitType> *hitv = (HitContainer<HitType>*)(params->hitv);
double *ncpv = (double*)(params->ncpv);
ModelType *mhp = (ModelType*)(params->mhp);
double *countv = (double*)(params->countv);
bool needCalcConPrb = model->getNeedCalcConPrb();
ReadType read;
READ_INT_TYPE N = hitv->getN();
double sum;
vector<double> fracs; //to remove this, do calculation twice
HIT_INT_TYPE fr, to, id;
if (needCalcConPrb || updateModel) { reader->reset(); }
if (updateModel) { mhp->init(); }
memset(countv, 0, sizeof(double) * (M + 1));
for (READ_INT_TYPE i = 0; i < N; i++) {
if (needCalcConPrb || updateModel) {
general_assert(reader->next(read), "Can not load a read!");
}
fr = hitv->getSAt(i);
to = hitv->getSAt(i + 1);
fracs.resize(to - fr + 1);
sum = 0.0;
if (needCalcConPrb) { ncpv[i] = model->getNoiseConPrb(read); }
fracs[0] = probv[0] * ncpv[i];
if (fracs[0] < EPSILON) fracs[0] = 0.0;
sum += fracs[0];
for (HIT_INT_TYPE j = fr; j < to; j++) {
HitType &hit = hitv->getHitAt(j);
if (needCalcConPrb) { hit.setConPrb(model->getConPrb(read, hit)); }
id = j - fr + 1;
fracs[id] = probv[hit.getSid()] * hit.getConPrb();
if (fracs[id] < EPSILON) fracs[id] = 0.0;
sum += fracs[id];
}
if (sum >= EPSILON) {
fracs[0] /= sum;
countv[0] += fracs[0];
if (updateModel) { mhp->updateNoise(read, fracs[0]); }
if (calcExpectedWeights) { ncpv[i] = fracs[0]; }
for (HIT_INT_TYPE j = fr; j < to; j++) {
HitType &hit = hitv->getHitAt(j);
id = j - fr + 1;
fracs[id] /= sum;
countv[hit.getSid()] += fracs[id];
if (updateModel) { mhp->update(read, hit, fracs[id]); }
if (calcExpectedWeights) { hit.setConPrb(fracs[id]); }
}
}
else if (calcExpectedWeights) {
ncpv[i] = 0.0;
for (HIT_INT_TYPE j = fr; j < to; j++) {
HitType &hit = hitv->getHitAt(j);
hit.setConPrb(0.0);
}
}
}
return NULL;
}
bool observeOneFullSweep(IoInputType& io,
const ModelType& model,
const PsimagLite::String& list,
bool hasTimeEvolution,
SizeType orbitals)
{
typedef typename ModelType::GeometryType GeometryType;
typedef Observer<VectorWithOffsetType,ModelType,IoInputType> ObserverType;
typedef ObservableLibrary<ObserverType> ObservableLibraryType;
const GeometryType& geometry = model.geometry();
bool verbose = false;
SizeType n = geometry.numberOfSites();
SizeType rows = n; // could be n/2 if there's enough symmetry
SizeType cols = n;
SizeType nf = n - 2;
SizeType trail = 0;
PsimagLite::Vector<PsimagLite::String>::Type vecOptions;
PsimagLite::tokenizer(list,vecOptions,",");
bool hasTrail = false;
for (SizeType i = 0; i < vecOptions.size(); ++i) {
PsimagLite::String item = vecOptions[i];
PsimagLite::String label = "%nf=";
std::size_t labelIndex = item.find(label);
if (labelIndex == 0) {
nf = atoi(item.substr(label.length()).c_str());
rows = nf;
cols = nf;
std::cerr<<"observe: Found "<<label<<" = "<<nf;
std::cerr<<" (rows and cols also set to it)\n";
}
label = "%trail=";
labelIndex = item.find(label);
if (labelIndex == 0) {
trail = atoi(item.substr(label.length()).c_str());
std::cerr<<"observe: Found "<<label<<" = "<<trail<<"\n";
hasTrail = true;
}
label = "%rows=";
labelIndex = item.find(label);
if (labelIndex == 0) {
std::cerr<<"observe: Found %rows= with index "<<labelIndex<<"\n";
rows = atoi(item.substr(label.length()).c_str());
}
label = "%cols=";
labelIndex = item.find(label);
if (labelIndex == 0) {
std::cerr<<"observe: Found %cols= with index "<<labelIndex<<"\n";
cols = atoi(item.substr(label.length()).c_str());
}
}
if (!hasTrail)
trail = n - 2 - nf;
ObservableLibraryType observerLib(io,
n,
hasTimeEvolution,
model,
nf,
trail,
verbose);
for (SizeType i = 0; i < vecOptions.size(); ++i) {
PsimagLite::String item = vecOptions[i];
if (item.find("%") == 0) continue;
if (item.length() > 0 && item[0] != '<')
observerLib.measureTriage(item,rows,cols,orbitals,hasTimeEvolution);
else
observerLib.interpret(item,rows,cols);
}
return observerLib.endOfData();
}
/**
* Checks equality of objects inside
* @param rhs - other wrapped value
* @return true, if wrapped objects are same
*/
bool operator==(const ModelType &rhs) const override {
return assetId() == rhs.assetId() and domainId() == rhs.domainId()
and precision() == rhs.precision();
}
bool observeOneFullSweep(
IoInputType& io,
const GeometryType& geometry,
const ModelType& model,
const PsimagLite::String& obsOptions,
bool hasTimeEvolution)
{
bool verbose = false;
typedef typename SparseMatrixType::value_type FieldType;
typedef Observer<FieldType,VectorWithOffsetType,ModelType,IoInputType>
ObserverType;
typedef ObservableLibrary<ObserverType,TargettingType> ObservableLibraryType;
SizeType n = geometry.numberOfSites();
//PsimagLite::String sSweeps = "sweeps=";
//PsimagLite::String::SizeTypeype begin = obsOptions.find(sSweeps);
//if (begin != PsimagLite::String::npos) {
// PsimagLite::String sTmp = obsOptions.substr(begin+sSweeps.length(),PsimagLite::String::npos);
//std::cout<<"sTmp="<<sTmp<<"\n";
// n = atoi(sTmp.c_str());
//}
ObservableLibraryType observerLib(io,n,hasTimeEvolution,model,verbose);
bool ot = false;
if (obsOptions.find("ot")!=PsimagLite::String::npos || obsOptions.find("time")!=PsimagLite::String::npos) ot = true;
if (hasTimeEvolution && ot) {
observerLib.measureTime("superDensity");
observerLib.measureTime("nupNdown");
observerLib.measureTime("nup+ndown");
if (obsOptions.find("sz")!=PsimagLite::String::npos) observerLib.measureTime("sz");
}
if (hasTimeEvolution) observerLib.setBrackets("time","time");
const PsimagLite::String& modelName = model.params().model;
SizeType rows = n; // could be n/2 if there's enough symmetry
// Immm supports only onepoint:
if (modelName=="Immm" && obsOptions!="onepoint") {
PsimagLite::String str(__FILE__);
str += " " + ttos(__LINE__) + "\n";
str += "Model Immm only supports onepoint\n";
throw PsimagLite::RuntimeError(str.c_str());
}
SizeType numberOfDofs = dofsFromModelName(model);
if (!hasTimeEvolution && obsOptions.find("onepoint")!=PsimagLite::String::npos) {
observerLib.measureTheOnePoints(numberOfDofs);
}
if (modelName.find("Heisenberg")==PsimagLite::String::npos) {
if (obsOptions.find("cc")!=PsimagLite::String::npos) {
observerLib.measure("cc",rows,n);
}
if (obsOptions.find("nn")!=PsimagLite::String::npos) {
observerLib.measure("nn",rows,n);
}
}
if (obsOptions.find("szsz")!=PsimagLite::String::npos) {
observerLib.measure("szsz",rows,n);
}
if (modelName.find("FeAsBasedSc")!=PsimagLite::String::npos ||
modelName.find("FeAsBasedScExtended")!=PsimagLite::String::npos ||
modelName.find("HubbardOneBand")!=PsimagLite::String::npos) {
bool dd4 = (obsOptions.find("dd4")!=PsimagLite::String::npos);
if (obsOptions.find("dd")!=PsimagLite::String::npos && !dd4) { // &&
//geometry.label(0).find("ladder")!=PsimagLite::String::npos) {
observerLib.measure("dd",rows,n);
}
// FOUR-POINT DELTA-DELTA^DAGGER:
if (dd4 && geometry.label(0).find("ladder")!=PsimagLite::String::npos) {
observerLib.measure("dd4",rows,n);
} // if dd4
}
if (modelName.find("HubbardOneBand")!=PsimagLite::String::npos &&
obsOptions.find("multi")!=PsimagLite::String::npos) {
observerLib.measure("multi",rows,n);
}
if (obsOptions.find("s+s-")!=PsimagLite::String::npos) {
observerLib.measure("s+s-",rows,n);
}
if (obsOptions.find("ss")!=PsimagLite::String::npos) {
observerLib.measure("ss",rows,n);
}
return observerLib.endOfData();
}
bool GetRolePermissions::operator==(const ModelType &rhs) const {
return roleId() == rhs.roleId();
}
bool processmodel(const string & inputmodelfile, int inputmodeltype, const string & outputmodelfile, int outputmodeltype, const string & corpusfile, PatternSetModel * constrainbymodel, IndexedCorpus * corpus, PatternModelOptions & options, bool continued, bool expand, int firstsentence, bool ignoreerrors, string inputmodelfile2, ClassDecoder * classdecoder, ClassEncoder * classencoder, bool print, bool report, bool nocoverage, bool histogram , bool query, string dorelations, bool doinstantiate, bool info, bool printreverseindex, int cooc, double coocthreshold, bool flexfromskip, const vector<string> & querypatterns) {
if (!(print || report || histogram || query || info || cooc || printreverseindex || (dorelations != "") || (!querypatterns.empty()) || (!outputmodelfile.empty()) )) {
cerr << "Ooops... You didn't really give me anything to do...that can't be right.. Please study the usage options (-h) again! Did you perhaps forget a --print or --outputmodel? " << endl;
return false;
}
ModelType * inputmodel;
string outputqualifier = "";
if ((outputmodeltype == UNINDEXEDPATTERNMODEL) || (outputmodeltype == UNINDEXEDPATTERNPOINTERMODEL)) {
outputqualifier += " unindexed";
}
if ((outputmodeltype == INDEXEDPATTERNPOINTERMODEL) || (outputmodeltype == UNINDEXEDPATTERNPOINTERMODEL)) {
outputqualifier += " pointer";
}
if (inputmodelfile.empty()) {
//train model from scratch
inputmodel = new ModelType(corpus);
cerr << "Training" << outputqualifier << " model on " << corpusfile <<endl;
inputmodel->train(corpusfile, options, constrainbymodel, NULL, continued,firstsentence,ignoreerrors);
if (constrainbymodel) {
cerr << "Unloading constraint model" << endl;
delete constrainbymodel;
constrainbymodel = NULL;
}
if (options.DOSKIPGRAMS) {
if ((inputmodeltype == UNINDEXEDPATTERNMODEL) || (inputmodeltype == UNINDEXEDPATTERNPOINTERMODEL)) {
cerr << "WARNING: Can't compute skipgrams non-exhaustively on unindexed model" << endl;
if (flexfromskip) cerr << "WARNING: Can't compute flexgrams from skipgrams on unindexed model" << endl;
} else {
if (!inputmodelfile2.empty()) cerr << "WARNING: Can not compute skipgrams constrained by " << inputmodelfile2 << "!" << endl;
if (!inputmodel->hasskipgrams) {
cerr << "Computing skipgrams" << endl;
inputmodel->trainskipgrams(options);
}
if (flexfromskip) {
cerr << "Computing flexgrams from skipgrams" << corpusfile <<endl;
int found = inputmodel->computeflexgrams_fromskipgrams();
cerr << found << " flexgrams found" << corpusfile <<endl;
}
}
}
} else {
//load model
cerr << "Loading pattern model " << inputmodelfile << " as" << outputqualifier << " model..."<<endl;
inputmodel = new ModelType(inputmodelfile, options, (PatternModelInterface*) constrainbymodel, corpus);
if ((corpus != NULL) && (inputmodel->hasskipgrams)) {
cerr << "Filtering skipgrams..." << endl;
inputmodel->pruneskipgrams(options.MINTOKENS, options.MINSKIPTYPES);
}
if ((!corpusfile.empty()) && (expand)) {
cerr << "Expanding model on " << corpusfile <<endl;
inputmodel->train(corpusfile, options, constrainbymodel,NULL, continued,firstsentence,ignoreerrors);
if (constrainbymodel) {
cerr << "Unloading constraint model" << endl;
delete constrainbymodel;
constrainbymodel = NULL;
}
} else if (options.DOSKIPGRAMS) {
if (constrainbymodel) {
cerr << "Unloading constraint model" << endl;
delete constrainbymodel;
constrainbymodel = NULL;
}
cerr << "Computing skipgrams" << endl;
if (!inputmodelfile2.empty()) cerr << "WARNING: Can not compute skipgrams constrained by " << inputmodelfile2 << "!" << endl;
inputmodel->trainskipgrams(options);
if (flexfromskip) {
cerr << "Computing flexgrams from skipgrams" << corpusfile <<endl;
int found = inputmodel->computeflexgrams_fromskipgrams();
cerr << found << " flexgrams found" << corpusfile <<endl;
}
} else {
if (constrainbymodel) {
cerr << "Unloading constraint model" << endl;
delete constrainbymodel;
constrainbymodel = NULL;
}
}
}
if (!outputmodelfile.empty()) {
cerr << "Writing model to " << outputmodelfile << endl;
inputmodel->write(outputmodelfile);
}
viewmodel<ModelType>(*inputmodel, classdecoder, classencoder, print, report, nocoverage, histogram, query, dorelations, doinstantiate, info, printreverseindex, cooc, coocthreshold);
if (!querypatterns.empty()) {
processquerypatterns<ModelType>(*inputmodel, classencoder, classdecoder, querypatterns, dorelations, doinstantiate);
}
delete inputmodel;
return true;
}
bool GetAssetInfo::operator==(const ModelType &rhs) const {
return assetId() == rhs.assetId();
}
bool BlocksQuery::operator==(const ModelType &rhs) const {
return creatorAccountId() == rhs.creatorAccountId()
and queryCounter() == rhs.queryCounter()
and createdTime() == rhs.createdTime()
and signatures() == rhs.signatures();
}
