float mmf::OptSO3MMFvMF::computeAssignment(uint32_t& N)
{
N = this->cld_.counts().sum();
// Compute log of the cluster weights and push them to GPU
Eigen::VectorXf pi = Eigen::VectorXf::Ones(K()*6)*1000;
std::cout << this->t_ << std::endl;
std::cout<<"counts: "<<this->cld_.counts().transpose()<<std::endl;
if (this->t_ == 0) {
pi.fill(1.);
} else if (this->t_ >24) {
std::cout << "truncating noisy MFs: " << std::endl;
for (uint32_t k=0; k<K()*6; ++k) {
float count = this->cld_.counts().middleRows((k/6)*6,6).sum();
pi(k) = count > 0.10*N ? count : 1.e-20;
std::cout << count << " < " << 0.1*N << std::endl;
}
} else {
// estimate the axis and MF proportions
// pi += this->cld_.counts();
// estimate only the MF proportions
for (uint32_t k=0; k<K()*6; ++k)
pi(k) += this->cld_.counts().middleRows((k/6)*6,6).sum();
if (estimateTau_) {
for (uint32_t k=0; k<K()*6; ++k)
if (this->cld_.counts()(k) == 0) {
taus_(k) = 0.; // uniform
} else {
Eigen::Vector3f mu = Eigen::Vector3f::Zero();
mu((k%6)/2) = (k%6)%2==0?-1.:1.;
mu = Rs_[k/6]*mu;
taus_(k) = jsc::vMF<3>::MLEstimateTau(
this->cld_.xSums().col(k).cast<double>(),
mu.cast<double>(), this->cld_.counts()(k));
}
} else {
taus_.fill(60.);
}
}
std::cout<<"pi: "<<pi.transpose()<<std::endl;
pi = (pi.array() / pi.sum()).array().log();
std::cout<<"pi: "<<pi.transpose()<<std::endl;
if (estimateTau_) {
std::cout << pi.transpose() << std::endl;
std::cout << taus_.transpose() << std::endl;
for (uint32_t k=0; k<K()*6; ++k) {
pi(k) -= jsc::vMF<3>::log2SinhOverZ(taus_(k)) - log(2.*M_PI);
}
}
pi_.set(pi);
Rot2Device();
Eigen::VectorXf residuals = Eigen::VectorXf::Zero(K()*6);
MMFvMFCostFctAssignmentGPU((float*)residuals.data(), d_cost, &N,
d_N_, cld_.d_x(), d_weights_, cld_.d_z(), d_mu_, pi_.data(),
cld_.N(), K());
return residuals.sum();
};
void FeatureEval::computeCorrelation(float * data, int vector_size, int size, std::vector<int> & big_to_small, int stride, int offset){
stride = stride ? stride : vector_size;
if(ll_->getSelection().size() == 0)
return;
std::set<uint16_t> labels;
for(boost::weak_ptr<Layer> wlayer: ll_->getSelection()){
for(uint16_t label : wlayer.lock()->getLabelSet())
labels.insert(label);
}
std::vector<float> layer = get_scaled_layer_mask(cl_->active_->labels_,
labels,
big_to_small,
size);
Eigen::MatrixXf correlation_mat = multi_correlate(layer, data, vector_size, size, stride, offset);
Eigen::MatrixXf Rxx = correlation_mat.topLeftCorner(vector_size, vector_size);
Eigen::VectorXf c = correlation_mat.block(0, vector_size, vector_size, 1);
//std::cout << correlation_mat << std::endl;
float R = c.transpose() * (Rxx.inverse() * c);
qDebug() << "R^2: " << R;
//qDebug() << "R: " << sqrt(R);
reportResult(R, c.data(), vector_size);
//Eigen::VectorXf tmp = (Rxx.inverse() * c);
qDebug() << "Y -> X correlation <<<<<<<<<<<<<";
std::cout << c << std::endl;
//qDebug() << "Coefs <<<<<<<<<<<<<";
//std::cout << tmp << std::endl;
}
void TraceStoreCol::SplineLossyCompress(const std::string &strategy, int order, int flow_ix, char *bad_wells,
float *mad, size_t num_rows, size_t num_cols, size_t num_frames, size_t num_flows,
int use_mesh_neighbors, size_t frame_stride, size_t flow_frame_stride, int16_t *data) {
Eigen::MatrixXf Basis;
vector<float> knots;
FillInKnots(strategy, num_frames, knots);
if (!knots.empty()) {
int boundaries[2];
boundaries[0] = 0;
boundaries[1] = num_frames;
basis_splines_endreps_local_v2(&knots[0], knots.size(), order, boundaries, sizeof(boundaries)/sizeof(boundaries[0]), Basis);
}
Eigen::MatrixXf SX = (Basis.transpose() * Basis).inverse() * Basis.transpose();
Eigen::MatrixXf Y(frame_stride, num_frames);
// Eigen::MatrixXf FlowMeans(num_flows, num_frames);
// FlowMeans.setZero();
int good_wells = 0;
char *bad_start = bad_wells;
char *bad_end = bad_start + frame_stride;
while(bad_start != bad_end) {
if (*bad_start++ == 0) {
good_wells++;
}
}
// if nothing good then skip it
if (good_wells < MIN_SAMPLE_WELL) {
return;
}
ChipReduction smoothed_avg;
int x_clip = num_cols;
int y_clip = num_rows;
if (use_mesh_neighbors == 0) {
x_clip = THUMBNAIL_SIZE;
y_clip = THUMBNAIL_SIZE;
}
smoothed_avg.Init(num_rows, num_cols, num_frames,
SMOOTH_REDUCE_STEP, SMOOTH_REDUCE_STEP,
y_clip, x_clip,
SMOOTH_REDUCE_STEP * SMOOTH_REDUCE_STEP * .4);
for (size_t frame_ix = 0; frame_ix < num_frames; frame_ix++) {
smoothed_avg.ReduceFrame(data + flow_ix * frame_stride + frame_ix * flow_frame_stride, bad_wells, frame_ix);
}
smoothed_avg.SmoothBlocks(SMOOTH_REDUCE_REGION, SMOOTH_REDUCE_REGION);
for (size_t frame_ix = 0; frame_ix < num_frames; frame_ix++) {
float *y_start = Y.data() + frame_ix * frame_stride;
float *y_end = y_start + frame_stride;
int16_t *trace_start = data + flow_ix * frame_stride + frame_ix * flow_frame_stride;
while(y_start != y_end) {
*y_start++ = *trace_start++;
}
}
// subtract off flow,frame avg
for (size_t frame_ix = 0; frame_ix < num_frames; frame_ix++) {
float *start = Y.data() + frame_stride * frame_ix;
float *end = start + frame_stride;
for (size_t row = 0; row < num_rows; row++) {
for (size_t col = 0; col < num_cols; col++) {
float avg = smoothed_avg.GetSmoothEst(row, col, frame_ix);
*start++ -= avg;
}
}
}
// Get coefficients to solve
Eigen::MatrixXf B = Y * SX.transpose();
// Uncompress data into yhat matrix
Eigen::MatrixXf Yhat = B * Basis.transpose();
// add the flow/frame averages back
for (size_t frame_ix = 0; frame_ix < num_frames; frame_ix++) {
float *start = Y.data() + frame_stride * frame_ix;
float *end = start + frame_stride;
float *hstart = Yhat.data() + frame_stride * frame_ix;
for (size_t row = 0; row < num_rows; row++) {
for (size_t col = 0; col < num_cols; col++) {
float avg = smoothed_avg.GetSmoothEst(row, col, frame_ix);
*start++ += avg;
*hstart++ += avg;
}
}
}
for (size_t frame_ix = 0; frame_ix < num_frames; frame_ix++) {
float *yhat_start = Yhat.data() + frame_ix * frame_stride;
float *yhat_end = yhat_start + frame_stride;
int16_t *trace_start = data + flow_ix * frame_stride + frame_ix * flow_frame_stride;
while(yhat_start != yhat_end) {
*trace_start++ = (int)(*yhat_start + .5f);
yhat_start++;
}
}
Y = Y - Yhat;
Eigen::VectorXf M = Y.rowwise().squaredNorm();
float *mad_start = mad;
float *mad_end = mad_start + frame_stride;
float *m_start = M.data();
while (mad_start != mad_end) {
*mad_start += *m_start;
mad_start++;
m_start++;
}
mad_start = mad;
mad_end = mad + frame_stride;
int norm_factor = num_frames;// num_flows * num_frames;
while (mad_start != mad_end) {
*mad_start /= norm_factor;
mad_start++;
}
}
void TraceStoreCol::SplineLossyCompress(const std::string &strategy, int order, char *bad_wells, float *mad) {
Eigen::MatrixXf Basis;
vector<float> knots;
FillInKnots(strategy, mFrames, knots);
// Eigen::Map<Eigen::MatrixXf, Eigen::Aligned> Basis(, compressed.n_frames, compressed.n_basis);
if (!knots.empty()) {
int boundaries[2];
boundaries[0] = 0;
boundaries[1] = mFrames;
basis_splines_endreps_local_v2(&knots[0], knots.size(), order, boundaries, sizeof(boundaries)/sizeof(boundaries[0]), Basis);
}
Eigen::MatrixXf SX = (Basis.transpose() * Basis).inverse() * Basis.transpose();
Eigen::MatrixXf Y(mFlowFrameStride, mFrames);
// Eigen::MatrixXf FlowMeans(mFlows, mFrames);
// FlowMeans.setZero();
int good_wells = 0;
char *bad_start = bad_wells;
char *bad_end = bad_start + mFrameStride;
while(bad_start != bad_end) {
if (*bad_start++ == 0) {
good_wells++;
}
}
// if nothing good then skip it
if (good_wells < MIN_SAMPLE_WELL) {
return;
}
std::vector<ChipReduction> smoothed_avg(mFlows);
int x_clip = mCols;
int y_clip = mRows;
if (mUseMeshNeighbors == 0) {
x_clip = THUMBNAIL_SIZE;
y_clip = THUMBNAIL_SIZE;
}
for (size_t flow_ix = 0; flow_ix < mFlows; flow_ix++) {
smoothed_avg[flow_ix].Init(mRows, mCols, mFrames,
SMOOTH_REDUCE_STEP, SMOOTH_REDUCE_STEP,
y_clip, x_clip,
SMOOTH_REDUCE_STEP * SMOOTH_REDUCE_STEP * .4);
for (size_t frame_ix = 0; frame_ix < mFrames; frame_ix++) {
smoothed_avg[flow_ix].ReduceFrame(&mData[0] + flow_ix * mFrameStride + frame_ix * mFlowFrameStride, bad_wells, frame_ix);
}
smoothed_avg[flow_ix].SmoothBlocks(SMOOTH_REDUCE_REGION, SMOOTH_REDUCE_REGION);
}
float *y_start = Y.data();
float *y_end = Y.data() + mData.size();
int16_t *trace_start = &mData[0];
while(y_start != y_end) {
*y_start++ = *trace_start++;
}
// // get the flow means per frame.
// for (size_t frame_ix = 0; frame_ix < mFrames; frame_ix++) {
// for(size_t flow_ix = 0; flow_ix < mFlows; flow_ix++) {
// float *start = Y.data() + mFlowFrameStride * frame_ix + flow_ix * mFrameStride;
// float *end = start + mFrameStride;
// float *sum = &FlowMeans(flow_ix, frame_ix);
// char *bad = bad_wells;
// while (start != end) {
// if (*bad == 0) {
// *sum += *start;
// }
// start++;
// bad++;
// }
// *sum /= good_wells;
// }
// }
// subtract off flow,frame avg
for (size_t frame_ix = 0; frame_ix < mFrames; frame_ix++) {
for(size_t flow_ix = 0; flow_ix < mFlows; flow_ix++) {
float *start = Y.data() + mFlowFrameStride * frame_ix + flow_ix * mFrameStride;
float *end = start + mFrameStride;
for (size_t row = 0; row < mRows; row++) {
for (size_t col = 0; col < mCols; col++) {
float avg = smoothed_avg[flow_ix].GetSmoothEst(row, col, frame_ix);
*start++ -= avg;
}
}
}
}
// // subtract them off
// Eigen::VectorXf col_mean = Y.colwise().sum();
// col_mean /= Y.rows();
// for (int i = 0; i < Y.cols(); i++) {
// Y.col(i).array() -= col_mean.coeff(i);
// }
// Get coefficients to solve
Eigen::MatrixXf B = Y * SX.transpose();
// Uncompress data into yhat matrix
Eigen::MatrixXf Yhat = B * Basis.transpose();
// add the flow/frame averages back
for (size_t frame_ix = 0; frame_ix < mFrames; frame_ix++) {
for(size_t flow_ix = 0; flow_ix < mFlows; flow_ix++) {
float *start = Y.data() + mFlowFrameStride * frame_ix + flow_ix * mFrameStride;
float *end = start + mFrameStride;
float *hstart = Yhat.data() + mFlowFrameStride * frame_ix + flow_ix * mFrameStride;
for (size_t row = 0; row < mRows; row++) {
for (size_t col = 0; col < mCols; col++) {
float avg = smoothed_avg[flow_ix].GetSmoothEst(row, col, frame_ix);
*start++ += avg;
*hstart++ += avg;
}
}
}
}
// for (size_t frame_ix = 0; frame_ix < mFrames; frame_ix++) {
// for(size_t flow_ix = 0; flow_ix < mFlows; flow_ix++) {
// float *start = Y.data() + mFlowFrameStride * frame_ix + flow_ix * mFrameStride;
// float *hstart = Yhat.data() + mFlowFrameStride * frame_ix + flow_ix * mFrameStride;
// float *end = start + mFrameStride;
// float avg = FlowMeans(flow_ix, frame_ix);
// while (start != end) {
// *start++ += avg;
// *hstart++ += avg;
// }
// }
// }
// for (int i = 0; i < Yhat.cols(); i++) {
// Yhat.col(i).array() += col_mean.coeff(i);
// Y.col(i).array() += col_mean.coeff(i);
// }
float *yhat_start = Yhat.data();
float *yhat_end = Yhat.data() + mData.size();
trace_start = &mData[0];
while(yhat_start != yhat_end) {
*trace_start++ = (int)(*yhat_start + .5);
yhat_start++;
}
Y = Y - Yhat;
Eigen::VectorXf M = Y.rowwise().squaredNorm();
for (size_t flow_ix = 0; flow_ix < mFlows; flow_ix++) {
float *mad_start = mad;
float *mad_end = mad_start + mFrameStride;
float *m_start = M.data() + flow_ix * mFrameStride;
while (mad_start != mad_end) {
*mad_start += *m_start;
mad_start++;
m_start++;
}
}
float *mad_start = mad;
float *mad_end = mad + mFrameStride;
int norm_factor = mFlows * mFrames;
while (mad_start != mad_end) {
*mad_start /= norm_factor;
mad_start++;
}
}
void apply(Ea& ea)
{
// boost::accumulators::accumulator_set<double, boost::accumulators::stats<boost::accumulators::tag::max> > acc_max;
// boost::accumulators::accumulator_set<double, boost::accumulators::stats<boost::accumulators::tag::min> > acc_min;
SFERES_CONST size_t k = Params::novelty::k;
// merge the population and the archive in a single archive
pop_t archive = _archive;
archive.insert(archive.end(), ea.pop().begin(), ea.pop().end());
// we compute all the distances from pop(i) to archive(j) and store them
Eigen::MatrixXf distances(ea.pop().size(), archive.size());
novelty::_distance_f<Phen> f(ea.pop(), archive, distances);
parallel::init();
parallel::p_for(parallel::range_t(0, ea.pop().size()), f);
// compute the sparseness of each individual of the population
// and potentially add to the archive
int added = 0;
for (size_t i = 0; i < ea.pop().size(); ++i) {
size_t nb_objs = ea.pop()[i]->fit().objs().size();
Eigen::VectorXf vd = distances.row(i);
// Get the k-nearest neighbors
double n = 0.0;
std::partial_sort(vd.data(),
vd.data() + k,
vd.data() + vd.size());
// Sum up the total of distances from k-nearest neighbors
// This is the novelty score
n = vd.head<k>().sum() / k;
// Set the novelty score to the last objective in the array of objectives
ea.pop()[i]->fit().set_obj(nb_objs - 1, n);
// std::cout << "rho = " << _rho_min << "; n = " << n << "\n";
// acc_max(n);
// acc_min(n);
// Check if this individual is already in the archive
if (!in_archive_already(ea.pop()[i]->id()) &&
(n > _rho_min // Check if the novelty score passes the threshold
|| misc::rand<float>() < Params::novelty::add_to_archive_prob))
{
// Add this individual to the permanent archive
_archive.push_back(ea.pop()[i]);
// Print this indiv just added
// std::cout << "----> n = " << n << "; label = " << ea.pop()[i]->fit().label() << "; score = " << ea.pop()[i]->fit().score() << "; id = " << ea.pop()[i]->id() <<"\n";
// std::cout << "----> n = " << n << "; label = " << "; id = " << ea.pop()[i]->id() <<"\n";
_not_added = 0;
++added;
} else {
// Do not add this individual to the archive
++_not_added;
}
} // end for all individuals
// update rho_min
if (_not_added > Params::novelty::stalled_tresh)//2500
{
_rho_min *= 0.95; // Lower rho_min
_not_added = 0; // After lowering the stalled threshold, we need to reset the _not_added count
}
if (_archive.size() > Params::novelty::k && added > Params::novelty::adding_tresh)//4
{
_rho_min *= 1.05f; // Increase rho_min
}
std::cout<<"a size:"<<_archive.size()<<std::endl;
// DEBUG
// std::cout << "a size:"<<_archive.size()
// << "; max: " << boost::accumulators::max(acc_max)
// << "; min: " << boost::accumulators::min(acc_min)
// << std::endl;
}
void ScaleMaps::run(const cv::Mat& img, const vector<FeatureType>& features,
vector<float>& scaleMap)
{
size_t r, c, i, j, nr, nc, min_row, max_row, min_col, max_col;
size_t neighborCount, coefficientCount = 0;
std::vector<Eigen::Triplet<float>> coefficients; // list of non-zeros coefficients
vector<float> weights(9);
float m, v, sum;
size_t pixels = img.total();
int index = -1, nindex;
// Initialization
vector<bool> scale(pixels, false);
for (i = 0; i < features.size(); ++i)
{
const FeatureType& feat = features[i];
scale[((int)feat.y)*img.cols + (int)feat.x] = true;
}
// Adjust image values
cv::Mat scaledImg = img.clone();
scaledImg += 1;
scaledImg *= (1 / 32.0f);
// For each pixel in the image
for (r = 0; r < scaledImg.rows; ++r)
{
min_row = (size_t)std::max(int(r - 1), 0);
max_row = (size_t)std::min(scaledImg.rows - 1, int(r + 1));
for (c = 0; c < scaledImg.cols; ++c)
{
// Increment pixel index
++index;
// If this is not a feature point
if (!scale[index])
{
min_col = (size_t)std::max(int(c - 1), 0);
max_col = (size_t)std::min(scaledImg.cols - 1, int(c + 1));
neighborCount = 0;
// Loop over 3x3 neighborhoods matrix
// and calculate the variance of the intensities
for (nr = min_row; nr <= max_row; ++nr)
{
for (nc = min_col; nc <= max_col; ++nc)
{
if (nr == r && nc == c) continue;
weights[neighborCount++] = scaledImg.at<float>(nr, nc);
}
}
weights[neighborCount] = scaledImg.at<float>(r, c);
// Calculate the weights statistics
getStat(weights, neighborCount + 1, m, v);
m *= 0.6;
if (v < EPS) v = EPS; // Avoid division by 0
// Apply weight function
mWeightFunc(weights, neighborCount, scaledImg.at<float>(r, c), m, v);
// Normalize the weights and set to coefficients
sum = std::accumulate(weights.begin(), weights.begin() + neighborCount, 0.0f);
i = 0;
for (nr = min_row; nr <= max_row; ++nr)
{
for (nc = min_col; nc <= max_col; ++nc)
{
if (nr == r && nc == c) continue;
nindex = nr*scaledImg.cols + nc;
coefficients.push_back(Eigen::Triplet<float>(
index, nindex, -weights[i++] / sum));
}
}
}
// Add center coefficient
coefficients.push_back(Eigen::Triplet<float>(index, index, 1));
}
}
// Build right side equation vector
Eigen::VectorXf b = Eigen::VectorXf::Zero(pixels);
for (i = 0; i < features.size(); ++i)
{
const FeatureType& feat = features[i];
b[((int)feat.y)*scaledImg.cols + (int)feat.x] = feat.scale;
}
// Build left side equation matrix
Eigen::SparseMatrix<float> A(pixels, pixels);
A.setFromTriplets(coefficients.begin(), coefficients.end());
/*/// Debug ///
std::ofstream file("Output.m");
cv::Mat_<int> rows(1, coefficients.size()), cols(1, coefficients.size());
cv::Mat_<float> values(1, coefficients.size());
for (i = 0; i < coefficients.size(); ++i)
{
rows.at<int>(i) = coefficients[i].row();
cols.at<int>(i) = coefficients[i].col();
values.at<float>(i) = coefficients[i].value();
}
file << "cpp_rows = " << rows << ";" << std::endl;
file << "cpp_cols = " << cols << ";" << std::endl;
file << "cpp_values = " << values << ";" << std::endl;
/////////////*/
// Solving
Eigen::SparseLU<Eigen::SparseMatrix<float>> slu(A);
Eigen::VectorXf x = slu.solve(b);
// Copy to output
scaleMap.resize(pixels);
memcpy(scaleMap.data(), x.data(), pixels*sizeof(float));
}
