void Postprocessing::applyFscWeighting(MultidimArray<Complex > &FT, MultidimArray<DOUBLE> my_fsc)
{
// Find resolution where fsc_true drops below zero for the first time
// Set all weights to zero beyond that resolution
int ires_max = 0 ;
FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY1D(my_fsc)
{
if (DIRECT_A1D_ELEM(my_fsc, i) < 1e-10)
break;
ires_max = i;
}
FOR_ALL_ELEMENTS_IN_FFTW_TRANSFORM(FT)
{
int ires = ROUND(sqrt((DOUBLE)kp * kp + ip * ip + jp * jp));
if (ires <= ires_max)
{
DOUBLE fsc = DIRECT_A1D_ELEM(my_fsc, ires);
if (fsc < 1e-10)
REPORT_ERROR("Postprocessing::applyFscWeighting BUG: fsc <= 0");
DIRECT_A3D_ELEM(FT, k, i, j) *= sqrt((2 * fsc) / (1 + fsc));
}
else
{
DIRECT_A3D_ELEM(FT, k, i, j) = 0.;
}
}
}
/* ------------------------------------------------------------------------- */
NaiveBayes::NaiveBayes(
const std::vector< MultidimArray<double> > &features,
const Matrix1D<double> &priorProbs,
int discreteLevels)
{
K = features.size();
Nfeatures=XSIZE(features[0]);
__priorProbsLog10.initZeros(K);
FOR_ALL_ELEMENTS_IN_MATRIX1D(__priorProbsLog10)
VEC_ELEM(__priorProbsLog10,i)=log10(VEC_ELEM(priorProbs,i));
// Create a dummy leaf for features that cannot classify
std::vector < MultidimArray<double> > aux(K);
dummyLeaf=new LeafNode(aux,0);
// Build a leafnode for each feature and assign a weight
__weights.initZeros(Nfeatures);
for (int f=0; f<Nfeatures; f++)
{
for (int k=0; k<K; k++)
features[k].getCol(f, aux[k]);
LeafNode *leaf=new LeafNode(aux,discreteLevels);
if (leaf->__discreteLevels>0)
{
__leafs.push_back(leaf);
DIRECT_A1D_ELEM(__weights,f)=__leafs[f]->computeWeight();
}
else
{
__leafs.push_back(dummyLeaf);
DIRECT_A1D_ELEM(__weights,f)=0;
delete leaf;
}
#ifdef DEBUG_WEIGHTS
if(debugging == true)
{
std::cout << "Node " << f << std::endl;
std::cout << *(__leafs[f]) << std::endl;
//char c;
//std::cin >> c;
}
#endif
}
double norm=__weights.computeMax();
if (norm>0)
__weights *= 1.0/norm;
// Set default cost matrix
__cost.resizeNoCopy(K,K);
__cost.initConstant(1);
for (int i=0; i<K; i++)
MAT_ELEM(__cost,i,i)=0;
}
void Postprocessing::makeGuinierPlot(MultidimArray<Complex > &FT, std::vector<fit_point2D> &guinier)
{
MultidimArray<int> radial_count(XSIZE(FT));
MultidimArray<DOUBLE> lnF(XSIZE(FT));
fit_point2D onepoint;
FOR_ALL_ELEMENTS_IN_FFTW_TRANSFORM(FT)
{
int r2 = kp * kp + ip * ip + jp * jp;
int ires = ROUND(sqrt((DOUBLE)r2));
if (ires < XSIZE(radial_count))
{
lnF(ires) += abs(DIRECT_A3D_ELEM(FT, k, i, j));
radial_count(ires)++;
}
}
DOUBLE xsize = XSIZE(I1());
guinier.clear();
FOR_ALL_ELEMENTS_IN_ARRAY1D(radial_count)
{
DOUBLE res = (xsize * angpix)/(DOUBLE)i; // resolution in Angstrom
if (res >= angpix * 2.) // Apply B-factor sharpening until Nyquist, then low-pass filter later on (with a soft edge)
{
onepoint.x = 1. / (res * res);
if (DIRECT_A1D_ELEM(lnF, i) > 0.)
{
onepoint.y = log ( DIRECT_A1D_ELEM(lnF, i) / DIRECT_A1D_ELEM(radial_count, i) );
if (res <= fit_minres && res >= fit_maxres)
{
onepoint.w = 1.;
}
else
{
onepoint.w = 0.;
}
}
else
{
onepoint.y = -99.;
onepoint.w = 0.;
}
//std::cerr << " onepoint.x= " << onepoint.x << " onepoint.y= " << onepoint.y << " onepoint.w= " << onepoint.w << std::endl;
guinier.push_back(onepoint);
}
}
}
// Value access
double TabFtBlob::operator()(double val) const
{
int idx = (int)( ABS(val) / sampling);
if (idx >= XSIZE(tabulatedValues))
return 0.;
else
return DIRECT_A1D_ELEM(tabulatedValues, idx);
}
void run()
{
// Get angles ==============================================================
MetaData angles;
angles.read(fnIn);
size_t AngleNo = angles.size();
if (AngleNo == 0 || !angles.containsLabel(MDL_ANGLE_ROT))
REPORT_ERROR(ERR_MD_BADLABEL, "Input file doesn't contain angular information");
double maxWeight = -99.e99;
MultidimArray<double> weight;
weight.initZeros(AngleNo);
if (angles.containsLabel(MDL_WEIGHT))
{
// Find maximum weight
int i=0;
FOR_ALL_OBJECTS_IN_METADATA(angles)
{
double w;
angles.getValue(MDL_WEIGHT,w,__iter.objId);
DIRECT_A1D_ELEM(weight,i++)=w;
maxWeight=XMIPP_MAX(w,maxWeight);
}
}
void Steerable::singleFilter(const MultidimArray<double>& Vin,
MultidimArray<double> &hx1, MultidimArray<double> &hy1, MultidimArray<double> &hz1,
MultidimArray<double> &Vout){
MultidimArray< std::complex<double> > H, Aux;
Vout.initZeros(Vin);
// Filter in X
#define MINUS_ONE_POWER(n) (((n)%2==0)? 1:-1)
FourierTransformer transformer;
transformer.FourierTransform(hx1,H);
FOR_ALL_ELEMENTS_IN_ARRAY1D(H)
H(i)*= MINUS_ONE_POWER(i);
FourierTransformer transformer2;
MultidimArray<double> aux(XSIZE(Vin));
transformer2.setReal(aux);
for (size_t k=0; k<ZSIZE(Vin); k++)
for (size_t i=0; i<YSIZE(Vin); i++)
{
for (size_t j=0; j<XSIZE(Vin); j++)
DIRECT_A1D_ELEM(aux,j)=DIRECT_A3D_ELEM(Vin,k,i,j);
transformer2.FourierTransform( );
transformer2.getFourierAlias( Aux );
Aux*=H;
transformer2.inverseFourierTransform( );
for (size_t j=0; j<XSIZE(Vin); j++)
DIRECT_A3D_ELEM(Vout,k,i,j)=XSIZE(aux)*DIRECT_A1D_ELEM(aux,j);
}
// Filter in Y
transformer.FourierTransform(hy1,H);
FOR_ALL_ELEMENTS_IN_ARRAY1D(H)
H(i)*= MINUS_ONE_POWER(i);
aux.initZeros(YSIZE(Vin));
transformer2.setReal(aux);
for (size_t k=0; k<ZSIZE(Vin); k++)
for (size_t j=0; j<XSIZE(Vin); j++)
{
for (size_t i=0; i<YSIZE(Vin); i++)
DIRECT_A1D_ELEM(aux,i)=DIRECT_A3D_ELEM(Vout,k,i,j);
transformer2.FourierTransform( );
transformer2.getFourierAlias( Aux );
Aux*=H;
transformer2.inverseFourierTransform( );
for (size_t i=0; i<YSIZE(Vin); i++)
DIRECT_A3D_ELEM(Vout,k,i,j)=XSIZE(aux)*DIRECT_A1D_ELEM(aux,i);
}
// Filter in Z
transformer.FourierTransform(hz1,H);
FOR_ALL_ELEMENTS_IN_ARRAY1D(H)
H(i)*= MINUS_ONE_POWER(i);
aux.initZeros(ZSIZE(Vin));
transformer2.setReal(aux);
for (size_t i=0; i<YSIZE(Vin); i++)
for (size_t j=0; j<XSIZE(Vin); j++)
{
for (size_t k=0; k<ZSIZE(Vin); k++)
DIRECT_A1D_ELEM(aux,k)=DIRECT_A3D_ELEM(Vout,k,i,j);
transformer2.FourierTransform( );
transformer2.getFourierAlias( Aux );
Aux*=H;
transformer2.inverseFourierTransform( );
for (size_t k=0; k<ZSIZE(Vin); k++)
DIRECT_A3D_ELEM(Vout,k,i,j)=XSIZE(aux)*DIRECT_A1D_ELEM(aux,k);
}
// If Missing wedge
if (MW!=NULL)
MW->removeWedge(Vout);
}
/** Kullback-Leibner divergence */
double getKullbackLeibnerDivergence(MultidimArray<Complex >& Fimg,
MultidimArray<Complex >& Fref, MultidimArray<double>& sigma2,
MultidimArray<double>& p_i, MultidimArray<double>& q_i, int highshell, int lowshell)
{
// First check dimensions are OK
if (!Fimg.sameShape(Fref))
{
REPORT_ERROR("getKullbackLeibnerDivergence ERROR: Fimg and Fref are not of the same shape.");
}
if (highshell < 0)
{
highshell = XSIZE(Fimg) - 1;
}
if (lowshell < 0)
{
lowshell = 0;
}
if (highshell > XSIZE(sigma2))
{
REPORT_ERROR("getKullbackLeibnerDivergence ERROR: highshell is larger than size of sigma2 array.");
}
if (highshell < lowshell)
{
REPORT_ERROR("getKullbackLeibnerDivergence ERROR: highshell is smaller than lowshell.");
}
// Initialize the histogram
MultidimArray<int> histogram;
int histogram_size = 101;
int histogram_origin = histogram_size / 2;
double sigma_max = 10.;
double histogram_factor = histogram_origin / sigma_max;
histogram.initZeros(histogram_size);
// This way this will work in both 2D and 3D
FOR_ALL_ELEMENTS_IN_FFTW_TRANSFORM(Fimg)
{
int ires = ROUND(sqrt(kp * kp + ip * ip + jp * jp));
if (ires >= lowshell && ires <= highshell)
{
// Use FT of masked image for noise estimation!
double diff_real = (DIRECT_A3D_ELEM(Fref, k, i, j)).real - (DIRECT_A3D_ELEM(Fimg, k, i, j)).real;
double diff_imag = (DIRECT_A3D_ELEM(Fref, k, i, j)).imag - (DIRECT_A3D_ELEM(Fimg, k, i, j)).imag;
double sigma = sqrt(DIRECT_A1D_ELEM(sigma2, ires));
// Divide by standard deviation to normalise all the difference
diff_real /= sigma;
diff_imag /= sigma;
// Histogram runs from -10 sigma to +10 sigma
diff_real += sigma_max;
diff_imag += sigma_max;
// Make histogram on-the-fly;
// Real part
int ihis = ROUND(diff_real * histogram_factor);
if (ihis < 0)
{
ihis = 0;
}
else if (ihis >= histogram_size)
{
ihis = histogram_size - 1;
}
histogram(ihis)++;
// Imaginary part
ihis = ROUND(diff_imag * histogram_factor);
if (ihis < 0)
{
ihis = 0;
}
else if (ihis > histogram_size)
{
ihis = histogram_size;
}
histogram(ihis)++;
}
}
// Normalise the histogram and the discretised analytical Gaussian
double norm = (double)histogram.sum();
double gaussnorm = 0.;
for (int i = 0; i < histogram_size; i++)
{
double x = (double)i / histogram_factor;
gaussnorm += gaussian1D(x - sigma_max, 1. , 0.);
}
// Now calculate the actual Kullback-Leibner divergence
double kl_divergence = 0.;
p_i.resize(histogram_size);
q_i.resize(histogram_size);
for (int i = 0; i < histogram_size; i++)
{
// Data distribution
p_i(i) = (double)histogram(i) / norm;
// Theoretical distribution
double x = (double)i / histogram_factor;
q_i(i) = gaussian1D(x - sigma_max, 1. , 0.) / gaussnorm;
if (p_i(i) > 0.)
{
kl_divergence += p_i(i) * log(p_i(i) / q_i(i));
}
}
kl_divergence /= (double)histogram_size;
return kl_divergence;
}
// Shift an image through phase-shifts in its Fourier Transform (without pretabulated sine and cosine)
void shiftImageInFourierTransform(MultidimArray<Complex >& in,
MultidimArray<Complex >& out,
double oridim, Matrix1D<double> shift)
{
out.resize(in);
shift /= -oridim;
double dotp, a, b, c, d, ac, bd, ab_cd, x, y, z, xshift, yshift, zshift;
switch (in.getDim())
{
case 1:
xshift = XX(shift);
if (ABS(xshift) < XMIPP_EQUAL_ACCURACY)
{
out = in;
return;
}
for (long int j = 0; j < XSIZE(in); j++)
{
x = j;
dotp = 2 * PI * (x * xshift);
a = cos(dotp);
b = sin(dotp);
c = DIRECT_A1D_ELEM(in, j).real;
d = DIRECT_A1D_ELEM(in, j).imag;
ac = a * c;
bd = b * d;
ab_cd = (a + b) * (c + d); // (ab_cd-ac-bd = ad+bc : but needs 4 multiplications)
DIRECT_A1D_ELEM(out, j) = Complex(ac - bd, ab_cd - ac - bd);
}
break;
case 2:
xshift = XX(shift);
yshift = YY(shift);
if (ABS(xshift) < XMIPP_EQUAL_ACCURACY && ABS(yshift) < XMIPP_EQUAL_ACCURACY)
{
out = in;
return;
}
for (long int i = 0; i < XSIZE(in); i++)
for (long int j = 0; j < XSIZE(in); j++)
{
x = j;
y = i;
dotp = 2 * PI * (x * xshift + y * yshift);
a = cos(dotp);
b = sin(dotp);
c = DIRECT_A2D_ELEM(in, i, j).real;
d = DIRECT_A2D_ELEM(in, i, j).imag;
ac = a * c;
bd = b * d;
ab_cd = (a + b) * (c + d);
DIRECT_A2D_ELEM(out, i, j) = Complex(ac - bd, ab_cd - ac - bd);
}
for (long int i = YSIZE(in) - 1; i >= XSIZE(in); i--)
{
y = i - YSIZE(in);
for (long int j = 0; j < XSIZE(in); j++)
{
x = j;
dotp = 2 * PI * (x * xshift + y * yshift);
a = cos(dotp);
b = sin(dotp);
c = DIRECT_A2D_ELEM(in, i, j).real;
d = DIRECT_A2D_ELEM(in, i, j).imag;
ac = a * c;
bd = b * d;
ab_cd = (a + b) * (c + d);
DIRECT_A2D_ELEM(out, i, j) = Complex(ac - bd, ab_cd - ac - bd);
}
}
break;
case 3:
xshift = XX(shift);
yshift = YY(shift);
zshift = ZZ(shift);
if (ABS(xshift) < XMIPP_EQUAL_ACCURACY && ABS(yshift) < XMIPP_EQUAL_ACCURACY && ABS(zshift) < XMIPP_EQUAL_ACCURACY)
{
out = in;
return;
}
for (long int k = 0; k < ZSIZE(in); k++)
{
z = (k < XSIZE(in)) ? k : k - ZSIZE(in);
for (long int i = 0; i < YSIZE(in); i++)
{
y = (i < XSIZE(in)) ? i : i - YSIZE(in);
for (long int j = 0; j < XSIZE(in); j++)
{
x = j;
dotp = 2 * PI * (x * xshift + y * yshift + z * zshift);
a = cos(dotp);
b = sin(dotp);
c = DIRECT_A3D_ELEM(in, k, i, j).real;
d = DIRECT_A3D_ELEM(in, k, i, j).imag;
ac = a * c;
bd = b * d;
ab_cd = (a + b) * (c + d);
DIRECT_A3D_ELEM(out, k, i, j) = Complex(ac - bd, ab_cd - ac - bd);
}
}
}
break;
default:
REPORT_ERROR("shiftImageInFourierTransform ERROR: dimension should be 1, 2 or 3!");
}
}
void ProgSortByStatistics::processInputPrepare(MetaData &SF)
{
PCAMahalanobisAnalyzer tempPcaAnalyzer;
tempPcaAnalyzer.clear();
Image<double> img;
MultidimArray<double> img2;
MultidimArray<int> radial_count;
MultidimArray<double> radial_avg;
Matrix1D<int> center(2);
center.initZeros();
if (verbose>0)
std::cout << " Processing training set ..." << std::endl;
int nr_imgs = SF.size();
if (verbose>0)
init_progress_bar(nr_imgs);
int c = XMIPP_MAX(1, nr_imgs / 60);
int imgno = 0, imgnoPCA=0;
MultidimArray<float> v;
MultidimArray<int> distance;
int dim;
bool thereIsEnable=SF.containsLabel(MDL_ENABLED);
bool first=true;
FOR_ALL_OBJECTS_IN_METADATA(SF)
{
if (thereIsEnable)
{
int enabled;
SF.getValue(MDL_ENABLED,enabled,__iter.objId);
if (enabled==-1)
continue;
}
img.readApplyGeo(SF,__iter.objId);
if (targetXdim!=-1 && targetXdim!=XSIZE(img()))
selfScaleToSize(LINEAR,img(),targetXdim,targetXdim,1);
MultidimArray<double> &mI=img();
mI.setXmippOrigin();
mI.statisticsAdjust(0,1);
// Overall statistics
Histogram1D hist;
compute_hist(mI,hist,-4,4,31);
// Radial profile
img2.resizeNoCopy(mI);
FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(img2)
{
double val=DIRECT_MULTIDIM_ELEM(mI,n);
DIRECT_MULTIDIM_ELEM(img2,n)=val*val;
}
if (first)
{
radialAveragePrecomputeDistance(img2, center, distance, dim);
first=false;
}
fastRadialAverage(img2, distance, dim, radial_avg, radial_count);
// Build vector
v.initZeros(XSIZE(hist)+XSIZE(img2)/2);
int idx=0;
FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY1D(hist)
v(idx++)=(float)DIRECT_A1D_ELEM(hist,i);
for (size_t i=0; i<XSIZE(img2)/2; i++)
v(idx++)=(float)DIRECT_A1D_ELEM(radial_avg,i);
tempPcaAnalyzer.addVector(v);
if (imgno % c == 0 && verbose>0)
progress_bar(imgno);
imgno++;
imgnoPCA++;
}
if (verbose>0)
progress_bar(nr_imgs);
MultidimArray<double> vavg,vstddev;
tempPcaAnalyzer.computeStatistics(vavg,vstddev);
tempPcaAnalyzer.evaluateZScore(2,20,false);
pcaAnalyzer.insert(pcaAnalyzer.begin(), tempPcaAnalyzer);
}
// Split several histograms within the indexes l0 and lF so that
// the entropy after division is maximized
int splitHistogramsUsingEntropy(const std::vector<Histogram1D> &hist,
size_t l0, size_t lF)
{
// Number of classes
int K = hist.size();
// Set everything outside l0 and lF to zero, and make it a PDF
std::vector<Histogram1D> histNorm;
for (int k = 0; k < K; k++)
{
Histogram1D histaux = hist[k];
for (size_t l = 0; l < XSIZE(histaux); l++)
if (l < l0 || l > lF)
DIRECT_A1D_ELEM(histaux,l) = 0;
histaux *= 1.0/histaux.sum();
histNorm.push_back(histaux);
}
// Compute for each class the probability of being l<=l0 and l>l0
MultidimArray<double> p(K, 2);
for (int k = 0; k < K; k++)
{
const Histogram1D& histogram=histNorm[k];
DIRECT_A2D_ELEM(p,k, 0) = DIRECT_A1D_ELEM(histogram,l0);
DIRECT_A2D_ELEM(p,k, 1) = 0;
for (size_t l = l0 + 1; l <= lF; l++)
DIRECT_A2D_ELEM(p,k, 1) += DIRECT_A1D_ELEM(histogram,l);
}
// Compute the splitting l giving maximum entropy
double maxEntropy = 0;
int lmaxEntropy = -1;
size_t l = l0;
while (l < lF)
{
// Compute the entropy of the classes if we split by l
double entropy = 0;
FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(p)
{
double aux=DIRECT_MULTIDIM_ELEM(p,n);
if (aux != 0)
entropy -= aux * log10(aux);
}
#ifdef DEBUG_SPLITTING_USING_ENTROPY
std::cout << "Splitting at " << l << " entropy=" << entropy
<< std::endl;
#endif
// Check if this is the maximum
if (entropy > maxEntropy)
{
maxEntropy = entropy;
lmaxEntropy = l;
}
// Move to next split point
++l;
// Update probabilities of being l<=l0 and l>l0
for (int k = 0; k < K; k++)
{
const Histogram1D& histogram=histNorm[k];
double aux=DIRECT_A1D_ELEM(histogram,l);
DIRECT_A2D_ELEM(p,k, 0) += aux;
DIRECT_A2D_ELEM(p,k, 1) -= aux;
}
}
#ifdef DEBUG_SPLITTING_USING_ENTROPY
std::cout << "Finally in l=[" << l0 << "," << lF
<< " Max Entropy:" << maxEntropy
<< " lmax=" << lmaxEntropy << std::endl;
#endif
// If the point giving the maximum entropy is too much on the extreme,
// substitute it by the middle point
if (lmaxEntropy<=2 || lmaxEntropy>=(int)lF-2)
lmaxEntropy = (int)ceil((lF + l0)/2.0);
return lmaxEntropy;
}
void Postprocessing::writeOutput()
{
if (verb > 0)
{
std::cout <<"== Writing out put files ..." <<std::endl;
}
// Write all relevant output information
FileName fn_tmp;
fn_tmp = fn_out + ".mrc";
DOUBLE avg, stddev, minval, maxval;
I1().computeStats(avg, stddev, minval, maxval);
I1.MDMainHeader.setValue(EMDL_IMAGE_STATS_MIN, minval);
I1.MDMainHeader.setValue(EMDL_IMAGE_STATS_MAX, maxval);
I1.MDMainHeader.setValue(EMDL_IMAGE_STATS_AVG, avg);
I1.MDMainHeader.setValue(EMDL_IMAGE_STATS_STDDEV, stddev);
I1.MDMainHeader.setValue(EMDL_IMAGE_SAMPLINGRATE_X, angpix);
I1.MDMainHeader.setValue(EMDL_IMAGE_SAMPLINGRATE_Y, angpix);
I1.MDMainHeader.setValue(EMDL_IMAGE_SAMPLINGRATE_Z, angpix);
I1.write(fn_tmp);
if (verb > 0)
{
std::cout.width(35); std::cout << std::left <<" + Processed map: "; std::cout << fn_tmp<< std::endl;
}
// Also write the masked postprocessed map
if (do_auto_mask || fn_mask != "")
{
fn_tmp = fn_out + "_masked.mrc";
I1() *= Im();
I1().computeStats(avg, stddev, minval, maxval);
I1.MDMainHeader.setValue(EMDL_IMAGE_STATS_MIN, minval);
I1.MDMainHeader.setValue(EMDL_IMAGE_STATS_MAX, maxval);
I1.MDMainHeader.setValue(EMDL_IMAGE_STATS_AVG, avg);
I1.MDMainHeader.setValue(EMDL_IMAGE_STATS_STDDEV, stddev);
I1.write(fn_tmp);
if (verb > 0)
{
std::cout.width(35); std::cout << std::left <<" + Processed masked map: "; std::cout << fn_tmp<< std::endl;
}
}
// Also write mask
if (do_auto_mask)
{
fn_tmp = fn_out + "_automask.mrc";
Im().computeStats(avg, stddev, minval, maxval);
Im.MDMainHeader.setValue(EMDL_IMAGE_STATS_MIN, minval);
Im.MDMainHeader.setValue(EMDL_IMAGE_STATS_MAX, maxval);
Im.MDMainHeader.setValue(EMDL_IMAGE_STATS_AVG, avg);
Im.MDMainHeader.setValue(EMDL_IMAGE_STATS_STDDEV, stddev);
Im.write(fn_tmp);
if (verb > 0)
{
std::cout.width(35); std::cout << std::left <<" + Auto-mask: "; std::cout << fn_tmp<< std::endl;
}
}
// Write an output STAR file with FSC curves, Guinier plots etc
std::ofstream fh;
fn_tmp = fn_out + ".star";
if (verb > 0)
{
std::cout.width(35); std::cout << std::left <<" + Metadata file: "; std::cout << fn_tmp<< std::endl;
}
fh.open((fn_tmp).c_str(), std::ios::out);
if (!fh)
REPORT_ERROR( (std::string)"MlOptimiser::write: Cannot write file: " + fn_tmp);
// Write the command line as a comment in the header
fh << "# RELION postprocess" << std::endl;
fh << "# ";
parser.writeCommandLine(fh);
MetaDataTable MDlist, MDfsc, MDguinier;
MDlist.setIsList(true);
MDlist.setName("general");
MDlist.addObject();
MDlist.setValue(EMDL_POSTPROCESS_FINAL_RESOLUTION, global_resol);
MDlist.setValue(EMDL_POSTPROCESS_BFACTOR, global_bfactor );
if (do_auto_bfac)
{
MDlist.setValue(EMDL_POSTPROCESS_GUINIER_FIT_SLOPE, global_slope);
MDlist.setValue(EMDL_POSTPROCESS_GUINIER_FIT_INTERCEPT, global_intercept);
MDlist.setValue(EMDL_POSTPROCESS_GUINIER_FIT_CORRELATION, global_corr_coeff);
}
MDlist.write(fh);
MDfsc.setName("fsc");
FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY1D(fsc_true)
{
MDfsc.addObject();
DOUBLE res = (i > 0) ? (XSIZE(I1()) * angpix / (DOUBLE)i) : 999.;
MDfsc.setValue(EMDL_SPECTRAL_IDX, (int)i);
MDfsc.setValue(EMDL_RESOLUTION, 1./res);
MDfsc.setValue(EMDL_RESOLUTION_ANGSTROM, res);
if (do_mask)
{
MDfsc.setValue(EMDL_POSTPROCESS_FSC_TRUE, DIRECT_A1D_ELEM(fsc_true, i) );
MDfsc.setValue(EMDL_POSTPROCESS_FSC_UNMASKED, DIRECT_A1D_ELEM(fsc_unmasked, i) );
MDfsc.setValue(EMDL_POSTPROCESS_FSC_MASKED, DIRECT_A1D_ELEM(fsc_masked, i) );
MDfsc.setValue(EMDL_POSTPROCESS_FSC_RANDOM_MASKED, DIRECT_A1D_ELEM(fsc_random_masked, i) );
}
else
{
MDfsc.setValue(EMDL_POSTPROCESS_FSC_UNMASKED, DIRECT_A1D_ELEM(fsc_true, i) );
}
}
MDfsc.write(fh);
// Also write XML file with FSC_true curve for EMDB submission
writeFscXml(MDfsc);
MDguinier.setName("guinier");
for (int i = 0; i < guinierin.size(); i++)
{
MDguinier.addObject();
MDguinier.setValue(EMDL_POSTPROCESS_GUINIER_RESOL_SQUARED, guinierin[i].x);
MDguinier.setValue(EMDL_POSTPROCESS_GUINIER_VALUE_IN, guinierin[i].y);
if (fn_mtf != "")
MDguinier.setValue(EMDL_POSTPROCESS_GUINIER_VALUE_INVMTF, guinierinvmtf[i].y);
if (do_fsc_weighting)
MDguinier.setValue(EMDL_POSTPROCESS_GUINIER_VALUE_WEIGHTED, guinierweighted[i].y);
if (do_auto_bfac || ABS(adhoc_bfac) > 0.)
MDguinier.setValue(EMDL_POSTPROCESS_GUINIER_VALUE_SHARPENED, guiniersharpen[i].y);
if (do_auto_bfac)
MDguinier.setValue(EMDL_POSTPROCESS_GUINIER_VALUE_INTERCEPT, global_intercept);
}
MDguinier.write(fh);
fh.close();
if (verb > 0)
{
std::cout.width(35); std::cout << std::left <<" + FINAL RESOLUTION: "; std::cout << global_resol<< std::endl;
}
}
void Postprocessing::divideByMtf(MultidimArray<Complex > &FT)
{
if (fn_mtf != "")
{
if (verb > 0)
{
std::cout << "== Dividing map by the MTF of the detector ..." << std::endl;
std::cout.width(35); std::cout << std::left <<" + mtf STAR-file: "; std::cout << fn_mtf << std::endl;
}
MetaDataTable MDmtf;
if (!fn_mtf.isStarFile())
REPORT_ERROR("Postprocessing::divideByMtf ERROR: input MTF file is not a STAR file.");
MDmtf.read(fn_mtf);
MultidimArray<DOUBLE> mtf_resol, mtf_value;
mtf_resol.resize(MDmtf.numberOfObjects());
mtf_value.resize(mtf_resol);
int i =0;
FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDmtf)
{
MDmtf.getValue(EMDL_RESOLUTION_INVPIXEL, DIRECT_A1D_ELEM(mtf_resol, i) ); // resolution needs to be given in 1/pix
MDmtf.getValue(EMDL_POSTPROCESS_MTF_VALUE, DIRECT_A1D_ELEM(mtf_value, i) );
if (DIRECT_A1D_ELEM(mtf_value, i) < 1e-10)
{
std::cerr << " i= " << i << " mtf_value[i]= " << DIRECT_A1D_ELEM(mtf_value, i) << std::endl;
REPORT_ERROR("Postprocessing::sharpenMap ERROR: zero or negative values encountered in MTF curve!");
}
i++;
}
DOUBLE xsize = (DOUBLE)XSIZE(I1());
FOR_ALL_ELEMENTS_IN_FFTW_TRANSFORM(FT)
{
int r2 = kp * kp + ip * ip + jp * jp;
DOUBLE res = sqrt((DOUBLE)r2)/xsize; // get resolution in 1/pixel
if (res < 0.5 )
{
// Find the suitable MTF value
int i_0 = 0;
for (int ii = 0; ii < XSIZE(mtf_resol); ii++)
{
if (DIRECT_A1D_ELEM(mtf_resol, ii) > res)
break;
i_0 = ii;
}
// linear interpolation: y = y_0 + (y_1 - y_0)*(x-x_0)/(x1_x0)
DOUBLE mtf;
DOUBLE x_0 = DIRECT_A1D_ELEM(mtf_resol, i_0);
if (i_0 == MULTIDIM_SIZE(mtf_resol) - 1 || i_0 == 0) // check boundaries of the array
mtf = DIRECT_A1D_ELEM(mtf_value, i_0);
else
{
DOUBLE x_1 = DIRECT_A1D_ELEM(mtf_resol, i_0 + 1);
DOUBLE y_0 = DIRECT_A1D_ELEM(mtf_value, i_0);
DOUBLE y_1 = DIRECT_A1D_ELEM(mtf_value, i_0 + 1);
mtf = y_0 + (y_1 - y_0)*(res - x_0)/(x_1 - x_0);
}
// Divide Fourier component by the MTF
DIRECT_A3D_ELEM(FT, k, i, j) /= mtf;
}
}
}
}
//majorAxis and minorAxis is the estimated particle size in px
void ProgSortByStatistics::processInprocessInputPrepareSPTH(MetaData &SF, bool trained)
{
//#define DEBUG
PCAMahalanobisAnalyzer tempPcaAnalyzer0;
PCAMahalanobisAnalyzer tempPcaAnalyzer1;
PCAMahalanobisAnalyzer tempPcaAnalyzer2;
PCAMahalanobisAnalyzer tempPcaAnalyzer3;
PCAMahalanobisAnalyzer tempPcaAnalyzer4;
//Morphology
tempPcaAnalyzer0.clear();
//Signal to noise ratio
tempPcaAnalyzer1.clear();
tempPcaAnalyzer2.clear();
tempPcaAnalyzer3.clear();
//Histogram analysis, to detect black points and saturated parts
tempPcaAnalyzer4.clear();
double sign = 1;//;-1;
int numNorm = 3;
int numDescriptors0=numNorm;
int numDescriptors2=4;
int numDescriptors3=11;
int numDescriptors4 = 10;
MultidimArray<float> v0(numDescriptors0);
MultidimArray<float> v2(numDescriptors2);
MultidimArray<float> v3(numDescriptors3);
MultidimArray<float> v4(numDescriptors4);
if (verbose>0)
{
std::cout << " Sorting particle set by new xmipp method..." << std::endl;
}
int nr_imgs = SF.size();
if (verbose>0)
init_progress_bar(nr_imgs);
int c = XMIPP_MAX(1, nr_imgs / 60);
int imgno = 0, imgnoPCA=0;
bool thereIsEnable=SF.containsLabel(MDL_ENABLED);
bool first=true;
// We assume that at least there is one particle
size_t Xdim, Ydim, Zdim, Ndim;
getImageSize(SF,Xdim,Ydim,Zdim,Ndim);
//Initialization:
MultidimArray<double> nI, modI, tempI, tempM, ROI;
MultidimArray<bool> mask;
nI.resizeNoCopy(Ydim,Xdim);
modI.resizeNoCopy(Ydim,Xdim);
tempI.resizeNoCopy(Ydim,Xdim);
tempM.resizeNoCopy(Ydim,Xdim);
mask.resizeNoCopy(Ydim,Xdim);
mask.initConstant(true);
MultidimArray<double> autoCorr(2*Ydim,2*Xdim);
MultidimArray<double> smallAutoCorr;
Histogram1D hist;
Matrix2D<double> U,V,temp;
Matrix1D<double> D;
MultidimArray<int> radial_count;
MultidimArray<double> radial_avg;
Matrix1D<int> center(2);
MultidimArray<int> distance;
int dim;
center.initZeros();
v0.initZeros(numDescriptors0);
v2.initZeros(numDescriptors2);
v3.initZeros(numDescriptors3);
v4.initZeros(numDescriptors4);
ROI.resizeNoCopy(Ydim,Xdim);
ROI.setXmippOrigin();
FOR_ALL_ELEMENTS_IN_ARRAY2D(ROI)
{
double temp = std::sqrt(i*i+j*j);
if ( temp < (Xdim/2))
A2D_ELEM(ROI,i,j)= 1;
else
A2D_ELEM(ROI,i,j)= 0;
}
Image<double> img;
FourierTransformer transformer(FFTW_BACKWARD);
FOR_ALL_OBJECTS_IN_METADATA(SF)
{
if (thereIsEnable)
{
int enabled;
SF.getValue(MDL_ENABLED,enabled,__iter.objId);
if ( (enabled==-1) )
{
imgno++;
continue;
}
}
img.readApplyGeo(SF,__iter.objId);
if (targetXdim!=-1 && targetXdim!=XSIZE(img()))
selfScaleToSize(LINEAR,img(),targetXdim,targetXdim,1);
MultidimArray<double> &mI=img();
mI.setXmippOrigin();
mI.statisticsAdjust(0,1);
mask.setXmippOrigin();
//The size of v1 depends on the image size and must be declared here
int numDescriptors1 = XSIZE(mI)/2; //=100;
MultidimArray<float> v1(numDescriptors1);
v1.initZeros(numDescriptors1);
double var = 1;
normalize(transformer,mI,tempI,modI,0,var,mask);
modI.setXmippOrigin();
tempI.setXmippOrigin();
nI = sign*tempI*(modI*modI);
tempM = (modI*modI);
A1D_ELEM(v0,0) = (tempM*ROI).sum();
int index = 1;
var+=2;
while (index < numNorm)
{
normalize(transformer,mI,tempI,modI,0,var,mask);
modI.setXmippOrigin();
tempI.setXmippOrigin();
nI += sign*tempI*(modI*modI);
tempM += (modI*modI);
A1D_ELEM(v0,index) = (tempM*ROI).sum();
index++;
var+=2;
}
nI /= tempM;
tempPcaAnalyzer0.addVector(v0);
nI=(nI*ROI);
auto_correlation_matrix(mI,autoCorr);
if (first)
{
radialAveragePrecomputeDistance(autoCorr, center, distance, dim);
first=false;
}
fastRadialAverage(autoCorr, distance, dim, radial_avg, radial_count);
for (int n = 0; n < numDescriptors1; ++n)
A1D_ELEM(v1,n)=(float)DIRECT_A1D_ELEM(radial_avg,n);
tempPcaAnalyzer1.addVector(v1);
#ifdef DEBUG
//String name = "000005@Images/Extracted/run_002/extra/BPV_1386.stk";
String name = "000010@Images/Extracted/run_001/extra/KLH_Dataset_I_Training_0028.stk";
//String name = "001160@Images/Extracted/run_001/DefaultFamily5";
std::cout << img.name() << std::endl;
if (img.name()==name2)
{
FileName fpName = "test_1.txt";
mI.write(fpName);
fpName = "test_2.txt";
nI.write(fpName);
fpName = "test_3.txt";
tempM.write(fpName);
fpName = "test_4.txt";
ROI.write(fpName);
//exit(1);
}
#endif
nI.binarize(0);
int im = labelImage2D(nI,nI,8);
compute_hist(nI, hist, 0, im, im+1);
size_t l;
int k,i,j;
hist.maxIndex(l,k,i,j);
A1D_ELEM(hist,j)=0;
hist.maxIndex(l,k,i,j);
nI.binarizeRange(j-1,j+1);
double x0=0,y0=0,majorAxis=0,minorAxis=0,ellipAng=0;
size_t area=0;
fitEllipse(nI,x0,y0,majorAxis,minorAxis,ellipAng,area);
A1D_ELEM(v2,0)=majorAxis/((img().xdim) );
A1D_ELEM(v2,1)=minorAxis/((img().xdim) );
A1D_ELEM(v2,2)= (fabs((img().xdim)/2-x0)+fabs((img().ydim)/2-y0))/((img().xdim)/2);
A1D_ELEM(v2,3)=area/( (double)((img().xdim)/2)*((img().ydim)/2) );
for (int n=0 ; n < numDescriptors2 ; n++)
{
if ( std::isnan(std::abs(A1D_ELEM(v2,n))))
A1D_ELEM(v2,n)=0;
}
tempPcaAnalyzer2.addVector(v2);
//mI.setXmippOrigin();
//auto_correlation_matrix(mI*ROI,autoCorr);
//auto_correlation_matrix(nI,autoCorr);
autoCorr.window(smallAutoCorr,-5,-5, 5, 5);
smallAutoCorr.copy(temp);
svdcmp(temp,U,D,V);
for (int n = 0; n < numDescriptors3; ++n)
A1D_ELEM(v3,n)=(float)VEC_ELEM(D,n); //A1D_ELEM(v3,n)=(float)VEC_ELEM(D,n)/VEC_ELEM(D,0);
tempPcaAnalyzer3.addVector(v3);
double minVal=0.;
double maxVal=0.;
mI.computeDoubleMinMax(minVal,maxVal);
compute_hist(mI, hist, minVal, maxVal, 100);
for (int n=0 ; n <= numDescriptors4-1 ; n++)
{
A1D_ELEM(v4,n)= (hist.percentil((n+1)*10));
}
tempPcaAnalyzer4.addVector(v4);
#ifdef DEBUG
if (img.name()==name1)
{
FileName fpName = "test.txt";
mI.write(fpName);
fpName = "test3.txt";
nI.write(fpName);
}
#endif
imgno++;
imgnoPCA++;
if (imgno % c == 0 && verbose>0)
progress_bar(imgno);
}
tempPcaAnalyzer0.evaluateZScore(2,20,trained);
tempPcaAnalyzer1.evaluateZScore(2,20,trained);
tempPcaAnalyzer2.evaluateZScore(2,20,trained);
tempPcaAnalyzer3.evaluateZScore(2,20,trained);
tempPcaAnalyzer4.evaluateZScore(2,20,trained);
pcaAnalyzer.push_back(tempPcaAnalyzer0);
pcaAnalyzer.push_back(tempPcaAnalyzer1);
pcaAnalyzer.push_back(tempPcaAnalyzer1);
pcaAnalyzer.push_back(tempPcaAnalyzer3);
pcaAnalyzer.push_back(tempPcaAnalyzer4);
}
void ProgSortByStatistics::run()
{
// Process input selfile ..............................................
SF.read(fn);
SF.removeDisabled();
MetaData SF2 = SF;
SF = SF2;
bool trained = false;
if (fn_train != "")
{
SFtrain.read(fn_train);
processInprocessInputPrepareSPTH(SFtrain,trained);
trained = true;
processInprocessInputPrepareSPTH(SF,trained);
}
else
processInprocessInputPrepareSPTH(SF,trained);
int imgno = 0;
int numPCAs = pcaAnalyzer.size();
MultidimArray<double> finalZscore(SF.size());
MultidimArray<double> ZscoreShape1(SF.size()), sortedZscoreShape1;
MultidimArray<double> ZscoreShape2(SF.size()), sortedZscoreShape2;
MultidimArray<double> ZscoreSNR1(SF.size()), sortedZscoreSNR1;
MultidimArray<double> ZscoreSNR2(SF.size()), sortedZscoreSNR2;
MultidimArray<double> ZscoreHist(SF.size()), sortedZscoreHist;
finalZscore.initConstant(0);
ZscoreShape1.resizeNoCopy(finalZscore);
ZscoreShape2.resizeNoCopy(finalZscore);
ZscoreSNR1.resizeNoCopy(finalZscore);
ZscoreSNR2.resizeNoCopy(finalZscore);
ZscoreHist.resizeNoCopy(finalZscore);
sortedZscoreShape1.resizeNoCopy(finalZscore);
sortedZscoreShape2.resizeNoCopy(finalZscore);
sortedZscoreSNR1.resizeNoCopy(finalZscore);
sortedZscoreSNR2.resizeNoCopy(finalZscore);
sortedZscoreHist.resizeNoCopy(finalZscore);
double zScore=0;
int enabled;
FOR_ALL_OBJECTS_IN_METADATA(SF)
{
SF.getValue(MDL_ENABLED,enabled,__iter.objId);
if ( (enabled==-1) )
{
A1D_ELEM(finalZscore,imgno) = 1e3;
A1D_ELEM(ZscoreShape1,imgno) = 1e3;
A1D_ELEM(ZscoreShape2,imgno) = 1e3;
A1D_ELEM(ZscoreSNR1,imgno) = 1e3;
A1D_ELEM(ZscoreSNR2,imgno) = 1e3;
A1D_ELEM(ZscoreHist,imgno) = 1e3;
imgno++;
enabled = 0;
}
else
{
for (int num = 0; num < numPCAs; ++num)
{
if (num == 0)
{
A1D_ELEM(ZscoreSNR1,imgno) = pcaAnalyzer[num].getZscore(imgno);
}
else if (num == 1)
{
A1D_ELEM(ZscoreShape2,imgno) = pcaAnalyzer[num].getZscore(imgno);
}
else if (num == 2)
{
A1D_ELEM(ZscoreShape1,imgno) = pcaAnalyzer[num].getZscore(imgno);
}
else if (num == 3)
{
A1D_ELEM(ZscoreSNR2,imgno) = pcaAnalyzer[num].getZscore(imgno);
}
else
{
A1D_ELEM(ZscoreHist,imgno) = pcaAnalyzer[num].getZscore(imgno);
}
if(zScore < pcaAnalyzer[num].getZscore(imgno))
zScore = pcaAnalyzer[num].getZscore(imgno);
}
A1D_ELEM(finalZscore,imgno)=zScore;
imgno++;
zScore = 0;
}
}
pcaAnalyzer.clear();
// Produce output .....................................................
MetaData SFout;
std::ofstream fh_zind;
if (verbose==2 && !fn_out.empty())
fh_zind.open((fn_out.withoutExtension() + "_vectors.xmd").c_str(), std::ios::out);
MultidimArray<int> sorted;
finalZscore.indexSort(sorted);
int nr_imgs = SF.size();
bool thereIsEnable=SF.containsLabel(MDL_ENABLED);
MDRow row;
for (int imgno = 0; imgno < nr_imgs; imgno++)
{
int isort_1 = DIRECT_A1D_ELEM(sorted,imgno);
int isort = isort_1 - 1;
SF.getRow(row, isort_1);
if (thereIsEnable)
{
int enabled;
row.getValue(MDL_ENABLED, enabled);
if (enabled==-1)
continue;
}
double zscore=DIRECT_A1D_ELEM(finalZscore,isort);
double zscoreShape1=DIRECT_A1D_ELEM(ZscoreShape1,isort);
double zscoreShape2=DIRECT_A1D_ELEM(ZscoreShape2,isort);
double zscoreSNR1=DIRECT_A1D_ELEM(ZscoreSNR1,isort);
double zscoreSNR2=DIRECT_A1D_ELEM(ZscoreSNR2,isort);
double zscoreHist=DIRECT_A1D_ELEM(ZscoreHist,isort);
DIRECT_A1D_ELEM(sortedZscoreShape1,imgno)=DIRECT_A1D_ELEM(ZscoreShape1,isort);
DIRECT_A1D_ELEM(sortedZscoreShape2,imgno)=DIRECT_A1D_ELEM(ZscoreShape2,isort);
DIRECT_A1D_ELEM(sortedZscoreSNR1,imgno)=DIRECT_A1D_ELEM(ZscoreSNR1,isort);
DIRECT_A1D_ELEM(sortedZscoreSNR2,imgno)=DIRECT_A1D_ELEM(ZscoreSNR2,isort);
DIRECT_A1D_ELEM(sortedZscoreHist,imgno)=DIRECT_A1D_ELEM(ZscoreHist,isort);
if (zscore>cutoff && cutoff>0)
{
row.setValue(MDL_ENABLED,-1);
if (addToInput)
SF.setValue(MDL_ENABLED,-1,isort_1);
}
else
{
row.setValue(MDL_ENABLED,1);
if (addToInput)
SF.setValue(MDL_ENABLED,1,isort_1);
}
row.setValue(MDL_ZSCORE,zscore);
row.setValue(MDL_ZSCORE_SHAPE1,zscoreShape1);
row.setValue(MDL_ZSCORE_SHAPE2,zscoreShape2);
row.setValue(MDL_ZSCORE_SNR1,zscoreSNR1);
row.setValue(MDL_ZSCORE_SNR2,zscoreSNR2);
row.setValue(MDL_ZSCORE_HISTOGRAM,zscoreHist);
if (addToInput)
{
SF.setValue(MDL_ZSCORE,zscore,isort_1);
SF.setValue(MDL_ZSCORE_SHAPE1,zscoreShape1,isort_1);
SF.setValue(MDL_ZSCORE_SHAPE2,zscoreShape2,isort_1);
SF.setValue(MDL_ZSCORE_SNR1,zscoreSNR1,isort_1);
SF.setValue(MDL_ZSCORE_SNR2,zscoreSNR2,isort_1);
SF.setValue(MDL_ZSCORE_HISTOGRAM,zscoreHist,isort_1);
}
SFout.addRow(row);
}
//Sorting taking into account a given percentage
if (per > 0)
{
MultidimArray<int> sortedShape1,sortedShape2,sortedSNR1,sortedSNR2,sortedHist,
sortedShapeSF1,sortedShapeSF2,sortedSNR1SF,sortedSNR2SF,sortedHistSF;
sortedZscoreShape1.indexSort(sortedShape1);
sortedZscoreShape2.indexSort(sortedShape2);
sortedZscoreSNR1.indexSort(sortedSNR1);
sortedZscoreSNR2.indexSort(sortedSNR2);
sortedZscoreHist.indexSort(sortedHist);
size_t numPartReject = (size_t)std::floor((per/100)*SF.size());
for (size_t numPar = SF.size()-1; numPar > (SF.size()-numPartReject); --numPar)
{
int isort_1 = DIRECT_A1D_ELEM(sortedShape1,numPar);
SFout.getRow(row, isort_1);
row.setValue(MDL_ENABLED,-1);
SFout.setRow(row,isort_1);
isort_1 = DIRECT_A1D_ELEM(sortedShape2,numPar);
SFout.getRow(row, isort_1);
row.setValue(MDL_ENABLED,-1);
SFout.setRow(row,isort_1);
isort_1 = DIRECT_A1D_ELEM(sortedSNR1,numPar);
SFout.getRow(row, isort_1);
row.setValue(MDL_ENABLED,-1);
SFout.setRow(row,isort_1);
isort_1 = DIRECT_A1D_ELEM(sortedSNR2,numPar);
SFout.getRow(row, isort_1);
row.setValue(MDL_ENABLED,-1);
SFout.setRow(row,isort_1);
isort_1 = DIRECT_A1D_ELEM(sortedHist,numPar);
SFout.getRow(row, isort_1);
row.setValue(MDL_ENABLED,-1);
SFout.setRow(row,isort_1);
if (addToInput)
{
ZscoreShape1.indexSort(sortedShapeSF1);
ZscoreShape2.indexSort(sortedShapeSF2);
ZscoreSNR1.indexSort(sortedSNR1SF);
ZscoreSNR2.indexSort(sortedSNR2SF);
ZscoreHist.indexSort(sortedHistSF);
isort_1 = DIRECT_A1D_ELEM(sortedShapeSF1,numPar);
SF.getRow(row, isort_1);
row.setValue(MDL_ENABLED,-1);
SF.setRow(row,isort_1);
isort_1 = DIRECT_A1D_ELEM(sortedShapeSF2,numPar);
SF.getRow(row, isort_1);
row.setValue(MDL_ENABLED,-1);
SF.setRow(row,isort_1);
isort_1 = DIRECT_A1D_ELEM(sortedSNR1SF,numPar);
SF.getRow(row, isort_1);
row.setValue(MDL_ENABLED,-1);
SF.setRow(row,isort_1);
isort_1 = DIRECT_A1D_ELEM(sortedSNR2SF,numPar);
SF.getRow(row, isort_1);
row.setValue(MDL_ENABLED,-1);
SF.setRow(row,isort_1);
isort_1 = DIRECT_A1D_ELEM(sortedHistSF,numPar);
SF.getRow(row, isort_1);
row.setValue(MDL_ENABLED,-1);
SF.setRow(row,isort_1);
}
}
}
if (verbose==2)
fh_zind.close();
if (!fn_out.empty())
{
MetaData SFsorted;
SFsorted.sort(SFout,MDL_ZSCORE);
SFout.write(fn_out,MD_OVERWRITE);
}
if (addToInput)
{
MetaData SFsorted;
SFsorted.sort(SF,MDL_ZSCORE);
SFsorted.write(fn,MD_APPEND);
}
}
void ParticlePolisherMpi::calculateAllSingleFrameReconstructionsAndBfactors()
{
FileName fn_star = fn_in.withoutExtension() + "_" + fn_out + "_bfactors.star";
if (!do_start_all_over && readStarFileBfactors(fn_star))
{
if (verb > 0)
std::cout << " + " << fn_star << " already exists: skipping calculation average of per-frame B-factors." <<std::endl;
return;
}
DOUBLE bfactor, offset, corr_coeff;
int total_nr_frames = last_frame - first_frame + 1;
long int my_first_frame, my_last_frame, my_nr_frames;
// Loop over all frames (two halves for each frame!) to be included in the reconstruction
// Each node does part of the work
divide_equally(2*total_nr_frames, node->size, node->rank, my_first_frame, my_last_frame);
my_nr_frames = my_last_frame - my_first_frame + 1;
if (verb > 0)
{
std::cout << " + Calculating per-frame reconstructions ... " << std::endl;
init_progress_bar(my_nr_frames);
}
for (long int i = my_first_frame; i <= my_last_frame; i++)
{
int iframe = (i >= total_nr_frames) ? i - total_nr_frames : i;
iframe += first_frame;
int ihalf = (i >= total_nr_frames) ? 2 : 1;
calculateSingleFrameReconstruction(iframe, ihalf);
if (verb > 0)
progress_bar(i - my_first_frame + 1);
}
if (verb > 0)
{
progress_bar(my_nr_frames);
}
MPI_Barrier(MPI_COMM_WORLD);
// Also calculate the average of all single-frames for both halves
if (node->rank == 0)
calculateAverageAllSingleFrameReconstructions(1);
else if (node->rank == 1)
calculateAverageAllSingleFrameReconstructions(2);
// Wait until all reconstructions have been done, and calculate the B-factors per-frame
MPI_Barrier(MPI_COMM_WORLD);
calculateBfactorSingleFrameReconstruction(-1, bfactor, offset, corr_coeff); // FSC between the two averages, also reads mask
MPI_Barrier(MPI_COMM_WORLD);
// Loop over all frames (two halves for each frame!) to be included in the reconstruction
// Each node does part of the work
divide_equally(total_nr_frames, node->size, node->rank, my_first_frame, my_last_frame);
my_nr_frames = my_last_frame - my_first_frame + 1;
if (verb > 0)
{
std::cout << " + Calculating per-frame B-factors ... " << std::endl;
init_progress_bar(my_nr_frames);
}
for (long int i = first_frame+my_first_frame; i <= first_frame+my_last_frame; i++)
{
calculateBfactorSingleFrameReconstruction(i, bfactor, offset, corr_coeff);
int iframe = i - first_frame;
DIRECT_A1D_ELEM(perframe_bfactors, iframe * 3 + 0) = bfactor;
DIRECT_A1D_ELEM(perframe_bfactors, iframe * 3 + 1) = offset;
DIRECT_A1D_ELEM(perframe_bfactors, iframe * 3 + 2) = corr_coeff;
if (verb > 0)
progress_bar(i - first_frame - my_first_frame + 1);
}
// Combine results from all nodes
MultidimArray<DOUBLE> allnodes_perframe_bfactors;
allnodes_perframe_bfactors.resize(perframe_bfactors);
MPI_Allreduce(MULTIDIM_ARRAY(perframe_bfactors), MULTIDIM_ARRAY(allnodes_perframe_bfactors), MULTIDIM_SIZE(perframe_bfactors), MY_MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
perframe_bfactors = allnodes_perframe_bfactors;
if (verb > 0)
{
progress_bar(my_nr_frames);
writeStarFileBfactors(fn_star);
// Also write a STAR file with the relative contributions of each frame to all frequencies
fn_star = fn_in.withoutExtension() + "_" + fn_out + "_relweights.star";
writeStarFileRelativeWeights(fn_star);
}
}
/* Assign probability ------------------------------------------------------ */
double LeafNode::assignProbability(double value, int k) const
{
const IrregularHistogram1D& hist=__leafPDF[k];
int index = hist.val2Index(value);
return DIRECT_A1D_ELEM(hist.__hist,index);
}
void Postprocessing::run()
{
// Read inout maps and perform some checks
initialise();
// Calculate FSC of the unmask maps
getFSC(I1(), I2(), fsc_unmasked);
// Check whether we'll do masking
do_mask = getMask();
if (do_mask)
{
if (verb > 0)
{
std::cout <<"== Masking input maps ..." <<std::endl;
}
// Mask I1 and I2 and calculated fsc_masked
I1() *= Im();
I2() *= Im();
getFSC(I1(), I2(), fsc_masked);
// To save memory re-read the same input maps again and randomize phases before masking
I1.read(fn_I1);
I2.read(fn_I2);
I1().setXmippOrigin();
I2().setXmippOrigin();
// Check at which resolution shell the FSC drops below randomize_fsc_at
int randomize_at = -1;
FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY1D(fsc_unmasked)
{
if (i > 0 && DIRECT_A1D_ELEM(fsc_unmasked, i) < randomize_fsc_at)
{
randomize_at = i;
break;
}
}
if (verb > 0)
{
std::cout.width(35); std::cout << std::left << " + randomize phases beyond: "; std::cout << XSIZE(I1())* angpix / randomize_at << " Angstroms" << std::endl;
}
if (randomize_at > 0)
{
randomizePhasesBeyond(I1(), randomize_at);
randomizePhasesBeyond(I2(), randomize_at);
// Mask randomized phases maps and calculated fsc_random_masked
I1() *= Im();
I2() *= Im();
getFSC(I1(), I2(), fsc_random_masked);
}
else
REPORT_ERROR("Postprocessing::run ERROR: FSC curve never drops below randomize_fsc_at. You may want to check your mask.");
// Now that we have fsc_masked and fsc_random_masked, calculate fsc_true according to Richard's formula
// FSC_true = FSC_t - FSC_n / ( )
fsc_true.resize(fsc_masked);
FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY1D(fsc_true)
{
// 29jan2015: let's move this 2 shells upwards, because of small artefacts near the resolution of randomisation!
if (i < randomize_at + 2)
{
DIRECT_A1D_ELEM(fsc_true, i) = DIRECT_A1D_ELEM(fsc_masked, i);
}
else
{
DOUBLE fsct = DIRECT_A1D_ELEM(fsc_masked, i);
DOUBLE fscn = DIRECT_A1D_ELEM(fsc_random_masked, i);
if (fscn > fsct)
DIRECT_A1D_ELEM(fsc_true, i) = 0.;
else
DIRECT_A1D_ELEM(fsc_true, i) = (fsct - fscn) / (1. - fscn);
}
}
// Now re-read the original maps yet again into memory
I1.read(fn_I1);
I2.read(fn_I2);
I1().setXmippOrigin();
I2().setXmippOrigin();
}
else
{
/* Do inference ------------------------------------------------------------ */
int NaiveBayes::doInference(const MultidimArray<double> &newFeatures, double &cost,
Matrix1D<double> &classesProbs, Matrix1D<double> &allCosts)
{
classesProbs=__priorProbsLog10;
for(int f=0; f<Nfeatures; f++)
{
const LeafNode &leaf_f=*(__leafs[f]);
double newFeatures_f=DIRECT_A1D_ELEM(newFeatures,f);
for (int k=0; k<K; k++)
{
double p = leaf_f.assignProbability(newFeatures_f, k);
if (fabs(p) < 1e-2)
VEC_ELEM(classesProbs,k) += -2*DIRECT_A1D_ELEM(__weights,f);
else
VEC_ELEM(classesProbs,k) += DIRECT_A1D_ELEM(__weights,f)*std::log10(p);
#ifdef DEBUG_FINE_CLASSIFICATION
if(debugging == true)
{
std::cout << "Feature " << f
<< " Probability for class " << k << " = "
<< classesProbs(k) << " increase= " << p
<< std::endl;
char c;
// COSS std::cin >> c;
// if (c=='q') debugging = false;
}
#endif
}
}
classesProbs-=classesProbs.computeMax();
// std::cout << "classesProbs " << classesProbs.transpose() << std::endl;
for (int k=0; k<K; k++)
VEC_ELEM(classesProbs,k)=pow(10.0,VEC_ELEM(classesProbs,k));
classesProbs*=1.0/classesProbs.sum();
// std::cout << "classesProbs norm " << classesProbs.transpose() << std::endl;
allCosts=__cost*classesProbs;
// std::cout << "allCosts " << allCosts.transpose() << std::endl;
int bestk=0;
cost=VEC_ELEM(allCosts,0)=std::log10(VEC_ELEM(allCosts,0));
for (int k=1; k<K; k++)
{
VEC_ELEM(allCosts,k)=std::log10(VEC_ELEM(allCosts,k));
if (VEC_ELEM(allCosts,k)<cost)
{
cost=VEC_ELEM(allCosts,k);
bestk=k;
}
}
#ifdef DEBUG_CLASSIFICATION
if(debugging == true)
{
for (int k=0; k<K; k++)
classesProbs(k)=log10(classesProbs(k));
std::cout << "Class probababilities=" << classesProbs.transpose()
<< "\n costs=" << allCosts.transpose()
<< " best class=" << bestk << " cost=" << cost << std::endl;
char c;
// COSS std::cin >> c;
// if (c=='q') debugging = false;
}
#endif
return bestk;
}
// Fill data array with oversampled Fourier transform, and calculate its power spectrum
void Projector::computeFourierTransformMap(MultidimArray<DOUBLE> &vol_in, MultidimArray<DOUBLE> &power_spectrum, int current_size, int nr_threads, bool do_gridding)
{
MultidimArray<DOUBLE> Mpad;
MultidimArray<Complex > Faux;
FourierTransformer transformer;
// DEBUGGING: multi-threaded FFTWs are giving me a headache?
// For a long while: switch them off!
//transformer.setThreadsNumber(nr_threads);
DOUBLE normfft;
// Size of padded real-space volume
int padoridim = padding_factor * ori_size;
// Initialize data array of the oversampled transform
ref_dim = vol_in.getDim();
// Make Mpad
switch (ref_dim)
{
case 2:
Mpad.initZeros(padoridim, padoridim);
normfft = (DOUBLE)(padding_factor * padding_factor);
break;
case 3:
Mpad.initZeros(padoridim, padoridim, padoridim);
if (data_dim ==3)
normfft = (DOUBLE)(padding_factor * padding_factor * padding_factor);
else
normfft = (DOUBLE)(padding_factor * padding_factor * padding_factor * ori_size);
break;
default:
REPORT_ERROR("Projector::computeFourierTransformMap%%ERROR: Dimension of the data array should be 2 or 3");
}
// First do a gridding pre-correction on the real-space map:
// Divide by the inverse Fourier transform of the interpolator in Fourier-space
// 10feb11: at least in 2D case, this seems to be the wrong thing to do!!!
// TODO: check what is best for subtomo!
if (do_gridding)// && data_dim != 3)
griddingCorrect(vol_in);
// Pad translated map with zeros
vol_in.setXmippOrigin();
Mpad.setXmippOrigin();
FOR_ALL_ELEMENTS_IN_ARRAY3D(vol_in) // This will also work for 2D
A3D_ELEM(Mpad, k, i, j) = A3D_ELEM(vol_in, k, i, j);
// Translate padded map to put origin of FT in the center
CenterFFT(Mpad, true);
// Calculate the oversampled Fourier transform
transformer.FourierTransform(Mpad, Faux, false);
// Free memory: Mpad no longer needed
Mpad.clear();
// Resize data array to the right size and initialise to zero
initZeros(current_size);
// Fill data only for those points with distance to origin less than max_r
// (other points will be zero because of initZeros() call above
// Also calculate radial power spectrum
power_spectrum.initZeros(ori_size / 2 + 1);
MultidimArray<DOUBLE> counter(power_spectrum);
counter.initZeros();
int max_r2 = r_max * r_max * padding_factor * padding_factor;
FOR_ALL_ELEMENTS_IN_FFTW_TRANSFORM(Faux) // This will also work for 2D
{
int r2 = kp*kp + ip*ip + jp*jp;
// The Fourier Transforms are all "normalised" for 2D transforms of size = ori_size x ori_size
if (r2 <= max_r2)
{
// Set data array
A3D_ELEM(data, kp, ip, jp) = DIRECT_A3D_ELEM(Faux, k, i, j) * normfft;
// Calculate power spectrum
int ires = ROUND( sqrt((DOUBLE)r2) / padding_factor );
// Factor two because of two-dimensionality of the complex plane
DIRECT_A1D_ELEM(power_spectrum, ires) += norm(A3D_ELEM(data, kp, ip, jp)) / 2.;
DIRECT_A1D_ELEM(counter, ires) += 1.;
}
}
// Calculate radial average of power spectrum
FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY1D(power_spectrum)
{
if (DIRECT_A1D_ELEM(counter, i) < 1.)
DIRECT_A1D_ELEM(power_spectrum, i) = 0.;
else
DIRECT_A1D_ELEM(power_spectrum, i) /= DIRECT_A1D_ELEM(counter, i);
}
transformer.cleanup();
}
EnsembleNaiveBayes::EnsembleNaiveBayes(
const std::vector < MultidimArray<double> > &features,
const Matrix1D<double> &priorProbs,
int discreteLevels, int numberOfClassifiers,
double samplingFeatures, double samplingIndividuals,
const std::string &newJudgeCombination)
{
int NFeatures=XSIZE(features[0]);
int NsubFeatures=CEIL(NFeatures*samplingFeatures);
K=features.size();
judgeCombination=newJudgeCombination;
#ifdef WEIGHTED_SAMPLING
// Measure the classification power of each variable
NaiveBayes *nb_weights=new NaiveBayes(features, priorProbs, discreteLevels);
MultidimArray<double> weights=nb_weights->__weights;
delete nb_weights;
double sumWeights=weights.sum();
#endif
for (int n=0; n<numberOfClassifiers; n++)
{
// Produce the set of features for this subclassifier
MultidimArray<int> subFeatures(NsubFeatures);
FOR_ALL_ELEMENTS_IN_ARRAY1D(subFeatures)
{
#ifdef WEIGHTED_SAMPLING
double random_sum_weight=rnd_unif(0,sumWeights);
int j=0;
do
{
double wj=DIRECT_A1D_ELEM(weights,j);
if (wj<random_sum_weight)
{
random_sum_weight-=wj;
j++;
if (j==NFeatures)
{
j=NFeatures-1;
break;
}
}
else
break;
}
while (true);
DIRECT_A1D_ELEM(subFeatures,i)=j;
#else
DIRECT_A1D_ELEM(subFeatures,i)=round(rnd_unif(0,NFeatures-1));
#endif
}
// Container for the new training sample
std::vector< MultidimArray<double> > newFeatures;
// Produce the data set for each class
for (int k=0; k<K; k++)
{
int NIndividuals=YSIZE(features[k]);
int NsubIndividuals=CEIL(NIndividuals*samplingIndividuals);
MultidimArray<int> subIndividuals(NsubIndividuals);
FOR_ALL_ELEMENTS_IN_ARRAY1D(subIndividuals)
subIndividuals(i)=ROUND(rnd_unif(0,NsubIndividuals-1));
MultidimArray<double> newFeaturesK;
newFeaturesK.initZeros(NsubIndividuals,NsubFeatures);
const MultidimArray<double>& features_k=features[k];
FOR_ALL_ELEMENTS_IN_ARRAY2D(newFeaturesK)
DIRECT_A2D_ELEM(newFeaturesK,i,j)=DIRECT_A2D_ELEM(features_k,
DIRECT_A1D_ELEM(subIndividuals,i),
DIRECT_A1D_ELEM(subFeatures,j));
newFeatures.push_back(newFeaturesK);
}
// Create a Naive Bayes classifier with this data
NaiveBayes *nb=new NaiveBayes(newFeatures, priorProbs, discreteLevels);
ensemble.push_back(nb);
ensembleFeatures.push_back(subFeatures);
}
}
void correctMapForMTF(MultidimArray<Complex >& FT, int ori_size, FileName& fn_mtf)
{
MetaDataTable MDmtf;
if (!fn_mtf.isStarFile())
{
REPORT_ERROR("correctMapForMTF ERROR: input MTF file is not a STAR file.");
}
MDmtf.read(fn_mtf);
MultidimArray<double> mtf_resol, mtf_value;
mtf_resol.resize(MDmtf.numberOfObjects());
mtf_value.resize(mtf_resol);
int i = 0;
FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDmtf)
{
MDmtf.getValue(EMDL_RESOLUTION_INVPIXEL, DIRECT_A1D_ELEM(mtf_resol, i)); // resolution needs to be given in 1/pix
MDmtf.getValue(EMDL_POSTPROCESS_MTF_VALUE, DIRECT_A1D_ELEM(mtf_value, i));
if (DIRECT_A1D_ELEM(mtf_value, i) < 1e-10)
{
std::cerr << " i= " << i << " mtf_value[i]= " << DIRECT_A1D_ELEM(mtf_value, i) << std::endl;
REPORT_ERROR("Postprocessing::sharpenMap ERROR: zero or negative values encountered in MTF curve!");
}
i++;
}
double xsize = (double)ori_size;
FOR_ALL_ELEMENTS_IN_FFTW_TRANSFORM(FT)
{
int r2 = kp * kp + ip * ip + jp * jp;
double res = sqrt((double)r2) / xsize; // get resolution in 1/pixel
if (res < 0.5)
{
// Find the suitable MTF value
int i_0 = 0;
for (int ii = 0; ii < XSIZE(mtf_resol); ii++)
{
if (DIRECT_A1D_ELEM(mtf_resol, ii) > res)
{
break;
}
i_0 = ii;
}
// linear interpolation: y = y_0 + (y_1 - y_0)*(x-x_0)/(x1_x0)
double mtf;
double x_0 = DIRECT_A1D_ELEM(mtf_resol, i_0);
if (i_0 == MULTIDIM_SIZE(mtf_resol) - 1 || i_0 == 0) // check boundaries of the array
{
mtf = DIRECT_A1D_ELEM(mtf_value, i_0);
}
else
{
double x_1 = DIRECT_A1D_ELEM(mtf_resol, i_0 + 1);
double y_0 = DIRECT_A1D_ELEM(mtf_value, i_0);
double y_1 = DIRECT_A1D_ELEM(mtf_value, i_0 + 1);
mtf = y_0 + (y_1 - y_0) * (res - x_0) / (x_1 - x_0);
}
// Divide Fourier component by the MTF
DIRECT_A3D_ELEM(FT, k, i, j) /= mtf;
}
}
}
