void ParticleSorterMpi::run() { int total_nr_images = MDin.numberOfObjects(); features.resize(total_nr_images, NR_FEATURES); // Each node does part of the work long int my_first_image, my_last_image, my_nr_images; divide_equally(total_nr_images, node->size, node->rank, my_first_image, my_last_image); my_nr_images = my_last_image - my_first_image + 1; int barstep; if (verb > 0) { std::cout << "Calculating sorting features for all input particles..." << std::endl; init_progress_bar(my_nr_images); barstep = XMIPP_MAX(1, my_nr_images/ 60); } long int ipart = 0; FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDin) { if (ipart >= my_first_image && ipart <= my_last_image) { if (verb > 0 && ipart % barstep == 0) progress_bar(ipart); calculateFeaturesOneParticle(ipart); } ipart++; } if (verb > 0) progress_bar(my_nr_images); // Combine results from all nodes MultidimArray<double> allnodes_features; allnodes_features.resize(features); MPI_Allreduce(MULTIDIM_ARRAY(features), MULTIDIM_ARRAY(allnodes_features), MULTIDIM_SIZE(features), MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); features = allnodes_features; // Only the master writes out files if (verb > 0) { normaliseFeatures(); writeFeatures(); } }
void ParticlePolisherMpi::polishParticlesAllMicrographs() { if (!do_start_all_over && exists(fn_out + ".star")) { if (verb > 0) std::cout << std::endl << " + " << fn_out << ".star already exists: skipping polishing of the particles." << std::endl; return; } int total_nr_micrographs = exp_model.average_micrographs.size(); // Each node does part of the work long int my_first_micrograph, my_last_micrograph, my_nr_micrographs; divide_equally(total_nr_micrographs, node->size, node->rank, my_first_micrograph, my_last_micrograph); my_nr_micrographs = my_last_micrograph - my_first_micrograph + 1; // Loop over all average micrographs int barstep; if (verb > 0) { std::cout << " + Write out polished particles for all micrographs ... " << std::endl; init_progress_bar(my_nr_micrographs); barstep = XMIPP_MAX(1, my_nr_micrographs/ 60); } for (long int i = my_first_micrograph; i <= my_last_micrograph; i++) { if (verb > 0 && i % barstep == 0) progress_bar(i); polishParticlesOneMicrograph(i); } if (verb > 0) progress_bar(my_nr_micrographs); if (node->isMaster()) writeStarFilePolishedParticles(); MPI_Barrier(MPI_COMM_WORLD); }
void run() { // Get angles ============================================================== MetaData angles; angles.read(fnIn); size_t AngleNo = angles.size(); if (AngleNo == 0 || !angles.containsLabel(MDL_ANGLE_ROT)) REPORT_ERROR(ERR_MD_BADLABEL, "Input file doesn't contain angular information"); double maxWeight = -99.e99; MultidimArray<double> weight; weight.initZeros(AngleNo); if (angles.containsLabel(MDL_WEIGHT)) { // Find maximum weight int i=0; FOR_ALL_OBJECTS_IN_METADATA(angles) { double w; angles.getValue(MDL_WEIGHT,w,__iter.objId); DIRECT_A1D_ELEM(weight,i++)=w; maxWeight=XMIPP_MAX(w,maxWeight); } }
void MpiProgImageRotationalPCA::createMutexes(size_t Nimgs) { fileMutex = new MpiFileMutex(node); threadMutex = new Mutex(); taskDistributor = new MpiTaskDistributor(Nimgs, XMIPP_MAX(1,Nimgs/(5*node->size)), node); }
void read(int argc, char **argv) { parser.setCommandLine(argc, argv); int general_section = parser.addSection("General Options"); fn_unt = parser.getOption("--u", "STAR file with the untilted xy-coordinates"); fn_til = parser.getOption("--t", "STAR file with the untilted xy-coordinates"); size = textToInteger(parser.getOption("--size", "Largest dimension of the micrograph (in pixels), e.g. 4096")); dim = textToInteger(parser.getOption("--dim", "Dimension of boxed particles (for EMAN .box files in pixels)", "200")); acc = textToFloat(parser.getOption("--acc", "Allowed accuracy (in pixels), e.g. half the particle diameter")); tilt = textToFloat(parser.getOption("--tilt", "Fix tilt angle (in degrees)", "99999.")); rot = textToFloat(parser.getOption("--rot", "Fix direction of the tilt axis (in degrees), 0 = along y, 90 = along x", "99999.")); do_opt = !parser.checkOption("--dont_opt", "Skip optimization of the transformation matrix"); mind2 = ROUND(acc * acc); int angle_section = parser.addSection("Specified tilt axis and translational search ranges"); tilt0 = textToFloat(parser.getOption("--tilt0", "Minimum tilt angle (in degrees)","0.")); tiltF = textToFloat(parser.getOption("--tiltF", "Maximum tilt angle (in degrees)","99999.")); if (tiltF == 99999.) tiltF = tilt0; tiltStep = textToFloat(parser.getOption("--tiltStep", "Tilt angle step size (in degrees)","1.")); rot0 = textToFloat(parser.getOption("--rot0", "Minimum rot angle (in degrees)","0.")); rotF = textToFloat(parser.getOption("--rotF", "Maximum rot angle (in degrees)","99999.")); if (rotF == 99999.) rotF = rot0; rotStep = textToFloat(parser.getOption("--rotStep", "Rot angle step size (in degrees)","1.")); x0 = textToInteger(parser.getOption("--x0", "Minimum X offset (pixels)","-99999")); xF = textToInteger(parser.getOption("--xF", "Maximum X offset (pixels)","99999")); xStep = textToInteger(parser.getOption("--xStep", "X offset step size (pixels)","-1")); y0 = textToInteger(parser.getOption("--y0", "Minimum Y offset (pixels)","-99999")); yF = textToInteger(parser.getOption("--yF", "Maximum Y offset (pixels)","99999")); yStep = textToInteger(parser.getOption("--yStep", "Y offset step size (pixels)","-1")); // Check for errors in the command-line option if (parser.checkForErrors()) REPORT_ERROR("Errors encountered on the command line, exiting..."); // If tilt and rot were given: do not search those if (tilt != 99999.) { tilt0 = tiltF = tilt; tiltStep = 1.; } if (rot != 99999.) { rot0 = rotF = rot; rotStep = 1.; } // By default search the entire micrograph x0 = XMIPP_MAX(x0, -size); xF = XMIPP_MIN(xF, size); // By default use a xStep of one third the accuracy if (xStep < 0) xStep = acc / 3; // By default treat y search in the same way as the x-search if (y0 == -99999) y0 = x0; if (yF == 99999) yF = xF; if (yStep < 0) yStep = xStep; // Done reading, now fill p_unt and p_til MDunt.read(fn_unt); MDtil.read(fn_til); // Check for the correct labels if (!MDunt.containsLabel(EMDL_IMAGE_COORD_X) || !MDunt.containsLabel(EMDL_IMAGE_COORD_Y)) REPORT_ERROR("ERROR: Untilted STAR file does not contain the rlnCoordinateX or Y labels"); if (!MDtil.containsLabel(EMDL_IMAGE_COORD_X) || !MDtil.containsLabel(EMDL_IMAGE_COORD_Y)) REPORT_ERROR("ERROR: Tilted STAR file does not contain the rlnCoordinateX or Y labels"); double x, y; p_unt.clear(); FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDunt) { MDunt.getValue(EMDL_IMAGE_COORD_X, x); MDunt.getValue(EMDL_IMAGE_COORD_Y, y); p_unt.push_back((int)x); p_unt.push_back((int)y); } p_til.clear(); FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDtil) { MDtil.getValue(EMDL_IMAGE_COORD_X, x); MDtil.getValue(EMDL_IMAGE_COORD_Y, y); p_til.push_back((int)x); p_til.push_back((int)y); } // Initialize best transformation params best_x = best_y = 9999; best_rot = best_tilt = 9999.; }
double optimiseTransformationMatrix(bool do_optimise_nr_pairs) { std::vector<int> best_pairs_t2u, best_map; double score, best_score, best_dist=9999.; if (do_optimise_nr_pairs) best_score = 0.; else best_score = -999999.; int nn = XMIPP_MAX(1., (rotF-rot0)/rotStep); nn *= XMIPP_MAX(1., (tiltF-tilt0)/tiltStep); nn *= XMIPP_MAX(1., (xF-x0)/xStep); nn *= XMIPP_MAX(1., (yF-y0)/yStep); int n = 0; init_progress_bar(nn); for (double rot = rot0; rot <= rotF; rot+= rotStep) { for (double tilt = tilt0; tilt <= tiltF; tilt+= tiltStep) { // Assume tilt-axis lies in-plane... double psi = -rot; // Rotate all points correspondingly Euler_angles2matrix(rot, tilt, psi, Pass); //std::cerr << " Pass= " << Pass << std::endl; // Zero-translations for now (these are added in the x-y loops below) MAT_ELEM(Pass, 0, 2) = MAT_ELEM(Pass, 1, 2) = 0.; mapOntoTilt(); for (int x = x0; x <= xF; x += xStep) { for (int y = y0; y <= yF; y += yStep, n++) { if (do_optimise_nr_pairs) score = getNumberOfPairs(x, y); else score = -getAverageDistance(x, y); // negative because smaller distance is better! bool is_best = false; if (do_optimise_nr_pairs && score==best_score) { double dist = getAverageDistance(x, y); if (dist < best_dist) { best_dist = dist; is_best = true; } } if (score > best_score || is_best) { best_score = score; best_pairs_t2u = pairs_t2u; best_rot = rot; best_tilt = tilt; best_x = x; best_y = y; } if (n%1000==0) progress_bar(n); } } } } progress_bar(nn); // Update pairs with the best_pairs if (do_optimise_nr_pairs) pairs_t2u = best_pairs_t2u; // Update the Passing matrix and the mapping Euler_angles2matrix(best_rot, best_tilt, -best_rot, Pass); // Zero-translations for now (these are added in the x-y loops below) MAT_ELEM(Pass, 0, 2) = MAT_ELEM(Pass, 1, 2) = 0.; mapOntoTilt(); return best_score; }
void ProgXrayImport::run() { // Delete output stack if it exists fnOut = fnRoot + ".mrc"; fnOut.deleteFile(); /* Turn off error handling */ H5Eset_auto(H5E_DEFAULT, NULL, NULL); if (dSource == MISTRAL) H5File.openFile(fnInput, H5F_ACC_RDONLY); // Reading bad pixels mask if ( !fnBPMask.empty() ) { std::cerr << "Reading bad pixels mask from "+fnBPMask << "." << std::endl; bpMask.read(fnBPMask); if ( (cropSizeX + cropSizeY ) > 0 ) bpMask().selfWindow(cropSizeY,cropSizeX, (int)(YSIZE(bpMask())-cropSizeY-1),(int)(XSIZE(bpMask())-cropSizeX-1)); STARTINGX(bpMask()) = STARTINGY(bpMask()) = 0; } // Setting the image projections list switch (dSource) { case MISTRAL: { inMD.read(fnInput); H5File.getDataset("NXtomo/data/rotation_angle", anglesArray, false); H5File.getDataset("NXtomo/instrument/sample/ExpTimes", expTimeArray, false); H5File.getDataset("NXtomo/instrument/sample/current", cBeamArray); /* In case there is no angles information we set them to to an increasing sequence * just to be able to continue importing data */ if ( anglesArray.size() != inMD.size() ) { reportWarning("Input file does not contains angle information. Default sequence used."); anglesArray.resizeNoCopy(inMD.size()); anglesArray.enumerate(); } // If expTime is empty or only one single value in nexus file then we fill with 1 if (expTimeArray.size() < 2) { reportWarning("Input file does not contains tomogram exposition time information."); expTimeArray.initConstant(anglesArray.size(), 1.); } // If current is empty or only one single value in nexus file then we fill with 1 if (cBeamArray.size() < 2) { reportWarning("Input file does not contains tomogram current beam information."); cBeamArray.initConstant(anglesArray.size(), 1.); } // Since Alba does not provide slit width, we set to ones slitWidthArray.initConstant(anglesArray.size(), 1.); } break; case BESSY: { size_t objId; for (size_t i = tIni; i <= tEnd; ++i) { objId = inMD.addObject(); inMD.setValue(MDL_IMAGE, fnInput + formatString("/img%d.spe", i), objId); } break; } case GENERIC: { // Get Darkfield std::cerr << "Getting darkfield from "+fnInput << " ..." << std::endl; getDarkfield(fnInput, IavgDark); if (XSIZE(IavgDark())!=0) IavgDark.write(fnRoot+"_darkfield.xmp"); std::vector<FileName> listDir; fnInput.getFiles(listDir); size_t objId; for (size_t i = 0; i < listDir.size(); ++i) { if (!listDir[i].hasImageExtension()) continue; objId = inMD.addObject(); inMD.setValue(MDL_IMAGE, fnInput+"/"+listDir[i], objId); } } break; } inMD.findObjects(objIds); size_t nIm = inMD.size(); // Create empty output stack file getImageInfo(inMD, imgInfo); /* Get the flatfield:: We get the FF after the image list because we need the image size to adapt the FF * in case they were already cropped. */ if (!fnFlat.empty()) { std::cout << "Getting flatfield from "+fnFlat << " ..." << std::endl; getFlatfield(fnFlat,IavgFlat); if ( XSIZE(IavgFlat()) != 0 ) { FileName ffName = fnRoot+"_flatfield_avg.xmp"; IavgFlat.write(ffName); fMD.setValue(MDL_IMAGE, ffName, fMD.addObject()); } } createEmptyFile(fnOut, imgInfo.adim.xdim-cropSizeXi-cropSizeXe, imgInfo.adim.ydim-cropSizeYi-cropSizeYe, 1, nIm); // Process images td = new ThreadTaskDistributor(nIm, XMIPP_MAX(1, nIm/30)); tm = new ThreadManager(thrNum, this); std::cerr << "Getting data from " << fnInput << " ...\n"; init_progress_bar(nIm); tm->run(runThread); progress_bar(nIm); // Write Metadata and angles MetaData MDSorted; MDSorted.sort(outMD,MDL_ANGLE_TILT); MDSorted.write("tomo@"+fnRoot + ".xmd"); if ( fMD.size() > 0 ) fMD.write("flatfield@"+fnRoot + ".xmd", MD_APPEND); // We also reference initial and final images at 0 degrees for Mistral tomograms if ( dSource == MISTRAL ) { fMD.clear(); FileName degree0Fn = "NXtomo/instrument/sample/0_degrees_initial_image"; if ( H5File.checkDataset(degree0Fn.c_str())) fMD.setValue(MDL_IMAGE, degree0Fn + "@" + fnInput, fMD.addObject()); degree0Fn = "NXtomo/instrument/sample/0_degrees_final_image"; if ( H5File.checkDataset(degree0Fn.c_str())) fMD.setValue(MDL_IMAGE, degree0Fn + "@" + fnInput, fMD.addObject()); if ( fMD.size() > 0 ) fMD.write("degree0@"+fnRoot + ".xmd", MD_APPEND); } // Write tlt file for IMOD std::ofstream fhTlt; fhTlt.open((fnRoot+".tlt").c_str()); if (!fhTlt) REPORT_ERROR(ERR_IO_NOWRITE,fnRoot+".tlt"); FOR_ALL_OBJECTS_IN_METADATA(MDSorted) { double tilt; MDSorted.getValue(MDL_ANGLE_TILT,tilt,__iter.objId); fhTlt << tilt << std::endl; } fhTlt.close(); delete td; delete tm; }
//#define DEBUG double spatial_Bspline03_proj( const Matrix1D<double> &r, const Matrix1D<double> &u) { // Avoids divisions by zero and allows orthogonal rays computation static Matrix1D<double> ur(3); if (XX(u) == 0) XX(ur) = XMIPP_EQUAL_ACCURACY; else XX(ur) = XX(u); if (YY(u) == 0) YY(ur) = XMIPP_EQUAL_ACCURACY; else YY(ur) = YY(u); if (ZZ(u) == 0) ZZ(ur) = XMIPP_EQUAL_ACCURACY; else ZZ(ur) = ZZ(u); // Some precalculated variables double x_sign = SGN(XX(ur)); double y_sign = SGN(YY(ur)); double z_sign = SGN(ZZ(ur)); // Compute the minimum and maximum alpha for the ray double alpha_xmin = (-2 - XX(r)) / XX(ur); double alpha_xmax = (2 - XX(r)) / XX(ur); double alpha_ymin = (-2 - YY(r)) / YY(ur); double alpha_ymax = (2 - YY(r)) / YY(ur); double alpha_zmin = (-2 - ZZ(r)) / ZZ(ur); double alpha_zmax = (2 - ZZ(r)) / ZZ(ur); double alpha_min = XMIPP_MAX(XMIPP_MIN(alpha_xmin, alpha_xmax), XMIPP_MIN(alpha_ymin, alpha_ymax)); alpha_min = XMIPP_MAX(alpha_min, XMIPP_MIN(alpha_zmin, alpha_zmax)); double alpha_max = XMIPP_MIN(XMIPP_MAX(alpha_xmin, alpha_xmax), XMIPP_MAX(alpha_ymin, alpha_ymax)); alpha_max = XMIPP_MIN(alpha_max, XMIPP_MAX(alpha_zmin, alpha_zmax)); if (alpha_max - alpha_min < XMIPP_EQUAL_ACCURACY) return 0.0; #ifdef DEBUG std::cout << "Pixel: " << r.transpose() << std::endl << "Dir: " << ur.transpose() << std::endl << "Alpha x:" << alpha_xmin << " " << alpha_xmax << std::endl << " " << (r + alpha_xmin*ur).transpose() << std::endl << " " << (r + alpha_xmax*ur).transpose() << std::endl << "Alpha y:" << alpha_ymin << " " << alpha_ymax << std::endl << " " << (r + alpha_ymin*ur).transpose() << std::endl << " " << (r + alpha_ymax*ur).transpose() << std::endl << "Alpha z:" << alpha_zmin << " " << alpha_zmax << std::endl << " " << (r + alpha_zmin*ur).transpose() << std::endl << " " << (r + alpha_zmax*ur).transpose() << std::endl << "alpha :" << alpha_min << " " << alpha_max << std::endl << std::endl; #endif // Compute the first point in the volume intersecting the ray static Matrix1D<double> v(3); V3_BY_CT(v, ur, alpha_min); V3_PLUS_V3(v, r, v); #ifdef DEBUG std::cout << "First entry point: " << v.transpose() << std::endl; std::cout << " Alpha_min: " << alpha_min << std::endl; #endif // Follow the ray double alpha = alpha_min; double ray_sum = 0; do { double alpha_x = (XX(v) + x_sign - XX(r)) / XX(ur); double alpha_y = (YY(v) + y_sign - YY(r)) / YY(ur); double alpha_z = (ZZ(v) + z_sign - ZZ(r)) / ZZ(ur); // Which dimension will ray move next step into?, it isn't neccesary to be only // one. double diffx = ABS(alpha - alpha_x); double diffy = ABS(alpha - alpha_y); double diffz = ABS(alpha - alpha_z); double diff_alpha = XMIPP_MIN(XMIPP_MIN(diffx, diffy), diffz); ray_sum += spatial_Bspline03_integral(r, ur, alpha, alpha + diff_alpha); // Update alpha and the next entry point if (ABS(diff_alpha - diffx) <= XMIPP_EQUAL_ACCURACY) alpha = alpha_x; if (ABS(diff_alpha - diffy) <= XMIPP_EQUAL_ACCURACY) alpha = alpha_y; if (ABS(diff_alpha - diffz) <= XMIPP_EQUAL_ACCURACY) alpha = alpha_z; XX(v) += diff_alpha * XX(ur); YY(v) += diff_alpha * YY(ur); ZZ(v) += diff_alpha * ZZ(ur); #ifdef DEBUG std::cout << "Alpha x,y,z: " << alpha_x << " " << alpha_y << " " << alpha_z << " ---> " << alpha << std::endl; std::cout << " Next entry point: " << v.transpose() << std::endl << " diff_alpha: " << diff_alpha << std::endl << " ray_sum: " << ray_sum << std::endl << " Alfa tot: " << alpha << "alpha_max: " << alpha_max << std::endl; #endif } while ((alpha_max - alpha) > XMIPP_EQUAL_ACCURACY); return ray_sum; }
// Fit the beam-induced translations for all average micrographs void ParticlePolisherMpi::fitMovementsAllMicrographs() { int total_nr_micrographs = exp_model.average_micrographs.size(); // Each node does part of the work long int my_first_micrograph, my_last_micrograph, my_nr_micrographs; divide_equally(total_nr_micrographs, node->size, node->rank, my_first_micrograph, my_last_micrograph); my_nr_micrographs = my_last_micrograph - my_first_micrograph + 1; // Loop over all average micrographs int barstep; if (verb > 0) { std::cout << " + Fitting straight paths for beam-induced movements in all micrographs ... " << std::endl; init_progress_bar(my_nr_micrographs); barstep = XMIPP_MAX(1, my_nr_micrographs/ 60); } for (long int i = my_first_micrograph; i <= my_last_micrograph; i++) { if (verb > 0 && i % barstep == 0) progress_bar(i); fitMovementsOneMicrograph(i); } // Wait until all micrographs have been done MPI_Barrier(MPI_COMM_WORLD); if (verb > 0) { progress_bar(my_nr_micrographs); } // Combine results from all nodes MultidimArray<DOUBLE> allnodes_fitted_movements; allnodes_fitted_movements.resize(fitted_movements); MPI_Allreduce(MULTIDIM_ARRAY(fitted_movements), MULTIDIM_ARRAY(allnodes_fitted_movements), MULTIDIM_SIZE(fitted_movements), MY_MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); fitted_movements = allnodes_fitted_movements; // Set the fitted movements in the xoff and yoff columns of the exp_model.MDimg for (long int ipart = 0; ipart < exp_model.numberOfParticles(); ipart++) { long int part_id = exp_model.particles[ipart].id; DOUBLE xoff = DIRECT_A2D_ELEM(fitted_movements, part_id, 0); DOUBLE yoff = DIRECT_A2D_ELEM(fitted_movements, part_id, 1); exp_model.MDimg.setValue(EMDL_ORIENT_ORIGIN_X, xoff, part_id); exp_model.MDimg.setValue(EMDL_ORIENT_ORIGIN_Y, yoff, part_id); } if (node->isMaster()) { // Write out the STAR file with all the fitted movements FileName fn_tmp = fn_in.withoutExtension() + "_" + fn_out + ".star"; exp_model.MDimg.write(fn_tmp); std::cout << " + Written out all fitted movements in STAR file: " << fn_tmp << std::endl; } }
void ParticlePolisherMpi::optimiseBeamTilt() { // This function assumes the shiny particles are in exp_mdel.MDimg!! if (beamtilt_max <= 0. && defocus_shift_max <= 0.) return; if (minres_beamtilt < maxres_model) { if (verb > 0) std::cout << " Skipping beamtilt correction, as the resolution of the shiny reconstruction does not go beyond minres_beamtilt of " << minres_beamtilt << " Ang." << std::endl; return; } getBeamTiltGroups(); initialiseSquaredDifferenceVectors(); int total_nr_micrographs = exp_model.micrographs.size(); // Each node does part of the work long int my_first_micrograph, my_last_micrograph, my_nr_micrographs; divide_equally(total_nr_micrographs, node->size, node->rank, my_first_micrograph, my_last_micrograph); my_nr_micrographs = my_last_micrograph - my_first_micrograph + 1; // Loop over all average micrographs int barstep; if (verb > 0) { std::cout << " + Optimising beamtilts and/or defocus values in all micrographs ... " << std::endl; init_progress_bar(my_nr_micrographs); barstep = XMIPP_MAX(1, my_nr_micrographs/ 60); } for (long int i = my_first_micrograph; i <= my_last_micrograph; i++) { if (verb > 0 && i % barstep == 0) progress_bar(i); optimiseBeamTiltAndDefocusOneMicrograph(i); } if (verb > 0) progress_bar(my_nr_micrographs); // Combine results from all nodes if (beamtilt_max > 0.) { MultidimArray<DOUBLE> allnodes_diff2_beamtilt; allnodes_diff2_beamtilt.initZeros(diff2_beamtilt); MPI_Allreduce(MULTIDIM_ARRAY(diff2_beamtilt), MULTIDIM_ARRAY(allnodes_diff2_beamtilt), MULTIDIM_SIZE(diff2_beamtilt), MY_MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); diff2_beamtilt = allnodes_diff2_beamtilt; } if (defocus_shift_max > 0.) { MultidimArray<DOUBLE> allnodes_defocus_shift_allmics; allnodes_defocus_shift_allmics.initZeros(defocus_shift_allmics); MPI_Allreduce(MULTIDIM_ARRAY(defocus_shift_allmics), MULTIDIM_ARRAY(allnodes_defocus_shift_allmics), MULTIDIM_SIZE(defocus_shift_allmics), MY_MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); defocus_shift_allmics = allnodes_defocus_shift_allmics; } // Now get the final optimised beamtilts and defocus shifts, and write results to the MetadataTable applyOptimisedBeamTiltsAndDefocus(); // Write the new MDTable to disc if (verb > 0) exp_model.MDimg.write(fn_out + ".star"); }
//majorAxis and minorAxis is the estimated particle size in px void ProgSortByStatistics::processInprocessInputPrepareSPTH(MetaData &SF, bool trained) { //#define DEBUG PCAMahalanobisAnalyzer tempPcaAnalyzer0; PCAMahalanobisAnalyzer tempPcaAnalyzer1; PCAMahalanobisAnalyzer tempPcaAnalyzer2; PCAMahalanobisAnalyzer tempPcaAnalyzer3; PCAMahalanobisAnalyzer tempPcaAnalyzer4; //Morphology tempPcaAnalyzer0.clear(); //Signal to noise ratio tempPcaAnalyzer1.clear(); tempPcaAnalyzer2.clear(); tempPcaAnalyzer3.clear(); //Histogram analysis, to detect black points and saturated parts tempPcaAnalyzer4.clear(); double sign = 1;//;-1; int numNorm = 3; int numDescriptors0=numNorm; int numDescriptors2=4; int numDescriptors3=11; int numDescriptors4 = 10; MultidimArray<float> v0(numDescriptors0); MultidimArray<float> v2(numDescriptors2); MultidimArray<float> v3(numDescriptors3); MultidimArray<float> v4(numDescriptors4); if (verbose>0) { std::cout << " Sorting particle set by new xmipp method..." << std::endl; } int nr_imgs = SF.size(); if (verbose>0) init_progress_bar(nr_imgs); int c = XMIPP_MAX(1, nr_imgs / 60); int imgno = 0, imgnoPCA=0; bool thereIsEnable=SF.containsLabel(MDL_ENABLED); bool first=true; // We assume that at least there is one particle size_t Xdim, Ydim, Zdim, Ndim; getImageSize(SF,Xdim,Ydim,Zdim,Ndim); //Initialization: MultidimArray<double> nI, modI, tempI, tempM, ROI; MultidimArray<bool> mask; nI.resizeNoCopy(Ydim,Xdim); modI.resizeNoCopy(Ydim,Xdim); tempI.resizeNoCopy(Ydim,Xdim); tempM.resizeNoCopy(Ydim,Xdim); mask.resizeNoCopy(Ydim,Xdim); mask.initConstant(true); MultidimArray<double> autoCorr(2*Ydim,2*Xdim); MultidimArray<double> smallAutoCorr; Histogram1D hist; Matrix2D<double> U,V,temp; Matrix1D<double> D; MultidimArray<int> radial_count; MultidimArray<double> radial_avg; Matrix1D<int> center(2); MultidimArray<int> distance; int dim; center.initZeros(); v0.initZeros(numDescriptors0); v2.initZeros(numDescriptors2); v3.initZeros(numDescriptors3); v4.initZeros(numDescriptors4); ROI.resizeNoCopy(Ydim,Xdim); ROI.setXmippOrigin(); FOR_ALL_ELEMENTS_IN_ARRAY2D(ROI) { double temp = std::sqrt(i*i+j*j); if ( temp < (Xdim/2)) A2D_ELEM(ROI,i,j)= 1; else A2D_ELEM(ROI,i,j)= 0; } Image<double> img; FourierTransformer transformer(FFTW_BACKWARD); FOR_ALL_OBJECTS_IN_METADATA(SF) { if (thereIsEnable) { int enabled; SF.getValue(MDL_ENABLED,enabled,__iter.objId); if ( (enabled==-1) ) { imgno++; continue; } } img.readApplyGeo(SF,__iter.objId); if (targetXdim!=-1 && targetXdim!=XSIZE(img())) selfScaleToSize(LINEAR,img(),targetXdim,targetXdim,1); MultidimArray<double> &mI=img(); mI.setXmippOrigin(); mI.statisticsAdjust(0,1); mask.setXmippOrigin(); //The size of v1 depends on the image size and must be declared here int numDescriptors1 = XSIZE(mI)/2; //=100; MultidimArray<float> v1(numDescriptors1); v1.initZeros(numDescriptors1); double var = 1; normalize(transformer,mI,tempI,modI,0,var,mask); modI.setXmippOrigin(); tempI.setXmippOrigin(); nI = sign*tempI*(modI*modI); tempM = (modI*modI); A1D_ELEM(v0,0) = (tempM*ROI).sum(); int index = 1; var+=2; while (index < numNorm) { normalize(transformer,mI,tempI,modI,0,var,mask); modI.setXmippOrigin(); tempI.setXmippOrigin(); nI += sign*tempI*(modI*modI); tempM += (modI*modI); A1D_ELEM(v0,index) = (tempM*ROI).sum(); index++; var+=2; } nI /= tempM; tempPcaAnalyzer0.addVector(v0); nI=(nI*ROI); auto_correlation_matrix(mI,autoCorr); if (first) { radialAveragePrecomputeDistance(autoCorr, center, distance, dim); first=false; } fastRadialAverage(autoCorr, distance, dim, radial_avg, radial_count); for (int n = 0; n < numDescriptors1; ++n) A1D_ELEM(v1,n)=(float)DIRECT_A1D_ELEM(radial_avg,n); tempPcaAnalyzer1.addVector(v1); #ifdef DEBUG //String name = "000005@Images/Extracted/run_002/extra/BPV_1386.stk"; String name = "000010@Images/Extracted/run_001/extra/KLH_Dataset_I_Training_0028.stk"; //String name = "001160@Images/Extracted/run_001/DefaultFamily5"; std::cout << img.name() << std::endl; if (img.name()==name2) { FileName fpName = "test_1.txt"; mI.write(fpName); fpName = "test_2.txt"; nI.write(fpName); fpName = "test_3.txt"; tempM.write(fpName); fpName = "test_4.txt"; ROI.write(fpName); //exit(1); } #endif nI.binarize(0); int im = labelImage2D(nI,nI,8); compute_hist(nI, hist, 0, im, im+1); size_t l; int k,i,j; hist.maxIndex(l,k,i,j); A1D_ELEM(hist,j)=0; hist.maxIndex(l,k,i,j); nI.binarizeRange(j-1,j+1); double x0=0,y0=0,majorAxis=0,minorAxis=0,ellipAng=0; size_t area=0; fitEllipse(nI,x0,y0,majorAxis,minorAxis,ellipAng,area); A1D_ELEM(v2,0)=majorAxis/((img().xdim) ); A1D_ELEM(v2,1)=minorAxis/((img().xdim) ); A1D_ELEM(v2,2)= (fabs((img().xdim)/2-x0)+fabs((img().ydim)/2-y0))/((img().xdim)/2); A1D_ELEM(v2,3)=area/( (double)((img().xdim)/2)*((img().ydim)/2) ); for (int n=0 ; n < numDescriptors2 ; n++) { if ( std::isnan(std::abs(A1D_ELEM(v2,n)))) A1D_ELEM(v2,n)=0; } tempPcaAnalyzer2.addVector(v2); //mI.setXmippOrigin(); //auto_correlation_matrix(mI*ROI,autoCorr); //auto_correlation_matrix(nI,autoCorr); autoCorr.window(smallAutoCorr,-5,-5, 5, 5); smallAutoCorr.copy(temp); svdcmp(temp,U,D,V); for (int n = 0; n < numDescriptors3; ++n) A1D_ELEM(v3,n)=(float)VEC_ELEM(D,n); //A1D_ELEM(v3,n)=(float)VEC_ELEM(D,n)/VEC_ELEM(D,0); tempPcaAnalyzer3.addVector(v3); double minVal=0.; double maxVal=0.; mI.computeDoubleMinMax(minVal,maxVal); compute_hist(mI, hist, minVal, maxVal, 100); for (int n=0 ; n <= numDescriptors4-1 ; n++) { A1D_ELEM(v4,n)= (hist.percentil((n+1)*10)); } tempPcaAnalyzer4.addVector(v4); #ifdef DEBUG if (img.name()==name1) { FileName fpName = "test.txt"; mI.write(fpName); fpName = "test3.txt"; nI.write(fpName); } #endif imgno++; imgnoPCA++; if (imgno % c == 0 && verbose>0) progress_bar(imgno); } tempPcaAnalyzer0.evaluateZScore(2,20,trained); tempPcaAnalyzer1.evaluateZScore(2,20,trained); tempPcaAnalyzer2.evaluateZScore(2,20,trained); tempPcaAnalyzer3.evaluateZScore(2,20,trained); tempPcaAnalyzer4.evaluateZScore(2,20,trained); pcaAnalyzer.push_back(tempPcaAnalyzer0); pcaAnalyzer.push_back(tempPcaAnalyzer1); pcaAnalyzer.push_back(tempPcaAnalyzer1); pcaAnalyzer.push_back(tempPcaAnalyzer3); pcaAnalyzer.push_back(tempPcaAnalyzer4); }
void ProgSortByStatistics::processInputPrepare(MetaData &SF) { PCAMahalanobisAnalyzer tempPcaAnalyzer; tempPcaAnalyzer.clear(); Image<double> img; MultidimArray<double> img2; MultidimArray<int> radial_count; MultidimArray<double> radial_avg; Matrix1D<int> center(2); center.initZeros(); if (verbose>0) std::cout << " Processing training set ..." << std::endl; int nr_imgs = SF.size(); if (verbose>0) init_progress_bar(nr_imgs); int c = XMIPP_MAX(1, nr_imgs / 60); int imgno = 0, imgnoPCA=0; MultidimArray<float> v; MultidimArray<int> distance; int dim; bool thereIsEnable=SF.containsLabel(MDL_ENABLED); bool first=true; FOR_ALL_OBJECTS_IN_METADATA(SF) { if (thereIsEnable) { int enabled; SF.getValue(MDL_ENABLED,enabled,__iter.objId); if (enabled==-1) continue; } img.readApplyGeo(SF,__iter.objId); if (targetXdim!=-1 && targetXdim!=XSIZE(img())) selfScaleToSize(LINEAR,img(),targetXdim,targetXdim,1); MultidimArray<double> &mI=img(); mI.setXmippOrigin(); mI.statisticsAdjust(0,1); // Overall statistics Histogram1D hist; compute_hist(mI,hist,-4,4,31); // Radial profile img2.resizeNoCopy(mI); FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(img2) { double val=DIRECT_MULTIDIM_ELEM(mI,n); DIRECT_MULTIDIM_ELEM(img2,n)=val*val; } if (first) { radialAveragePrecomputeDistance(img2, center, distance, dim); first=false; } fastRadialAverage(img2, distance, dim, radial_avg, radial_count); // Build vector v.initZeros(XSIZE(hist)+XSIZE(img2)/2); int idx=0; FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY1D(hist) v(idx++)=(float)DIRECT_A1D_ELEM(hist,i); for (size_t i=0; i<XSIZE(img2)/2; i++) v(idx++)=(float)DIRECT_A1D_ELEM(radial_avg,i); tempPcaAnalyzer.addVector(v); if (imgno % c == 0 && verbose>0) progress_bar(imgno); imgno++; imgnoPCA++; } if (verbose>0) progress_bar(nr_imgs); MultidimArray<double> vavg,vstddev; tempPcaAnalyzer.computeStatistics(vavg,vstddev); tempPcaAnalyzer.evaluateZScore(2,20,false); pcaAnalyzer.insert(pcaAnalyzer.begin(), tempPcaAnalyzer); }
// Evaluate plane ---------------------------------------------------------- double evaluatePlane(double rot, double tilt, const MultidimArray<double> *V, const MultidimArray<double> *Vmag, double maxFreq, double planeWidth, int direction, MultidimArray<double> *Vdraw=NULL, bool setPos=false, double rotPos=0, double tiltPos=0) { if (rot<0 || rot>360 || tilt<-90 || tilt>90) return 0; Matrix2D<double> E, Einv; Euler_angles2matrix(rot,tilt,0,E); Einv=E.transpose(); if (setPos) { Matrix2D<double> Epos; Euler_angles2matrix(rotPos,tiltPos,0,Epos); double angle=acos(E(2,0)*Epos(2,0)+E(2,1)*Epos(2,1)+E(2,2)*Epos(2,2)); angle=RAD2DEG(angle); if (fabs(angle)<20 || fabs(180-angle)<20) return 0; } size_t N=XMIPP_MAX(XSIZE(*Vmag),YSIZE(*Vmag)/2); N=XMIPP_MAX(N,ZSIZE(*Vmag)/2); double df=0.5/N; Matrix1D<double> freq(3), freqp(3); Matrix1D<int> idx(3); double sumNeg=0, sumPos=0; int Nneg=0, Npos=0; double maxFreq2=maxFreq*maxFreq; int iPlaneWidth=(int)ceil(planeWidth); for (double ix=0; ix<=N; ix++) { XX(freq)=ix*df; double fx2=XX(freq)*XX(freq); if (fx2>maxFreq2) continue; for (double iy=-(int)N; iy<=N; iy++) { YY(freq)=iy*df; double fx2fy2=fx2+YY(freq)*YY(freq); if (fx2fy2>maxFreq2) continue; for (int iz=-iPlaneWidth; iz<=iPlaneWidth; iz++) { if (iz==0 || ix==0 || iy==0) continue; // Frequency in the coordinate system of the plane ZZ(freq)=iz*df; // Frequency in the coordinate system of the volume SPEED_UP_temps012; M3x3_BY_V3x1(freqp,Einv,freq); bool inverted=false; if (XX(freqp)<0) { XX(freqp)=-XX(freqp); YY(freqp)=-YY(freqp); ZZ(freqp)=-ZZ(freqp); inverted=true; } // Get the corresponding index DIGFREQ2FFT_IDX(ZZ(freqp), ZSIZE(*V), ZZ(idx)); DIGFREQ2FFT_IDX(YY(freqp), YSIZE(*V), YY(idx)); DIGFREQ2FFT_IDX(XX(freqp), XSIZE(*V), XX(idx)); if (XX(idx) < STARTINGX(*Vmag) || XX(idx) > FINISHINGX(*Vmag) || YY(idx) < STARTINGY(*Vmag) || YY(idx) > FINISHINGY(*Vmag) || ZZ(idx) < STARTINGZ(*Vmag) || ZZ(idx) > FINISHINGZ(*Vmag)) continue; // Make the corresponding sums bool negativeSum; if (direction==1) negativeSum=iz<0; else negativeSum=iz>0; double val=A3D_ELEM(*Vmag,ZZ(idx),YY(idx),XX(idx)); if ((negativeSum && !inverted) || (!negativeSum && inverted)) // XOR { sumNeg+=val; Nneg++; if (Vdraw!=NULL) (*Vdraw)(idx)=2*direction*val; } else { sumPos+=val; Npos++; if (Vdraw!=NULL) (*Vdraw)(idx)=1.0/2.0*direction*val; } } } } if (fabs(Nneg-Npos)/(0.5*(Nneg+Npos))>0.5) // If there is a difference of more than 50% return 1e38; if (Nneg!=0) sumNeg/=Nneg; else return 1e38; if (Npos!=0) sumPos/=Npos; else return 1e38; return -(sumPos-sumNeg); }
void ProgSSNR::estimateSSNR(int dim, Matrix2D<double> &output) { // These vectors are for 1D Matrix1D<double> S_S21D((int)(XSIZE(S()) / 2 - ring_width)), S_N21D((int)(XSIZE(S()) / 2 - ring_width)), K1D((int)(XSIZE(S()) / 2 - ring_width)), S_SSNR1D; Matrix1D<double> N_S21D((int)(XSIZE(S()) / 2 - ring_width)), N_N21D((int)(XSIZE(S()) / 2 - ring_width)), N_SSNR1D; // Selfile of the 2D images MetaData SF_individual; std::cerr << "Computing the SSNR ...\n"; init_progress_bar(SF_S.size()); int imgno = 1; Image<double> Is, In; Projection Iths, Ithn; MultidimArray< std::complex<double> > FFT_Is, FFT_Iths, FFT_In, FFT_Ithn; MultidimArray<double> S2s, N2s, S2n, N2n; FileName fn_img; FourierTransformer FT(FFTW_BACKWARD); FourierProjector *Sprojector=NULL; FourierProjector *Nprojector=NULL; if (fourierProjections) { Sprojector=new FourierProjector(S(),2,0.5,LINEAR); Nprojector=new FourierProjector(N(),2,0.5,LINEAR); } FOR_ALL_OBJECTS_IN_METADATA2(SF_S, SF_N) { double rot, tilt, psi; SF_S.getValue(MDL_ANGLE_ROT,rot, __iter.objId); SF_S.getValue(MDL_ANGLE_TILT,tilt,__iter.objId); SF_S.getValue(MDL_ANGLE_PSI,psi,__iter.objId); SF_S.getValue(MDL_IMAGE,fn_img,__iter.objId); Is.read(fn_img); Is().setXmippOrigin(); SF_N.getValue(MDL_IMAGE,fn_img,__iter2.objId); In.read(fn_img); In().setXmippOrigin(); if (fourierProjections) { projectVolume(*Sprojector, Iths, YSIZE(Is()), XSIZE(Is()), rot, tilt, psi); projectVolume(*Nprojector, Ithn, YSIZE(Is()), XSIZE(Is()), rot, tilt, psi); } else { projectVolume(S(), Iths, YSIZE(Is()), XSIZE(Is()), rot, tilt, psi); projectVolume(N(), Ithn, YSIZE(Is()), XSIZE(Is()), rot, tilt, psi); } #ifdef DEBUG Image<double> save; save() = Is(); save.write("PPPread_signal.xmp"); save() = In(); save.write("PPPread_noise.xmp"); save() = Iths(); save.write("PPPtheo_signal.xmp"); save() = Ithn(); save.write("PPPtheo_noise.xmp"); #endif Is() -= Iths(); In() -= Ithn(); // According to the article: should we not subtract here (simply remove this line) // "...except that there is no subtraction in the denominator because the // underlying signal is zero by definition." if (dim == 2) { FT.completeFourierTransform(Is(), FFT_Is); FT.completeFourierTransform(Iths(), FFT_Iths); FT.completeFourierTransform(In(), FFT_In); FT.completeFourierTransform(Ithn(), FFT_Ithn); } else { FT.FourierTransform(Is(), FFT_Is); FT.FourierTransform(Iths(), FFT_Iths); FT.FourierTransform(In(), FFT_In); FT.FourierTransform(Ithn(), FFT_Ithn); } #ifdef DEBUG Image< std::complex<double> > savec; savec() = FFT_Is; savec.write("PPPFFTread_signal.xmp"); savec() = FFT_In; savec.write("PPPFFTread_noise.xmp"); savec() = FFT_Iths; savec.write("PPPFFTtheo_signal.xmp"); savec() = FFT_Ithn; savec.write("PPPFFTtheo_noise.xmp"); #endif // Compute the amplitudes S2s.resizeNoCopy(FFT_Iths); N2s.resizeNoCopy(FFT_Iths); S2n.resizeNoCopy(FFT_Iths); N2n.resizeNoCopy(FFT_Iths); FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(FFT_Iths) { DIRECT_MULTIDIM_ELEM(S2s, n) = abs(DIRECT_MULTIDIM_ELEM(FFT_Iths, n)); DIRECT_MULTIDIM_ELEM(S2s, n) *= DIRECT_MULTIDIM_ELEM(S2s, n); DIRECT_MULTIDIM_ELEM(N2s, n) = abs(DIRECT_MULTIDIM_ELEM(FFT_Is, n)); DIRECT_MULTIDIM_ELEM(N2s, n) *= DIRECT_MULTIDIM_ELEM(N2s, n); DIRECT_MULTIDIM_ELEM(S2n, n) = abs(DIRECT_MULTIDIM_ELEM(FFT_Ithn, n)); DIRECT_MULTIDIM_ELEM(S2n, n) *= DIRECT_MULTIDIM_ELEM(S2n, n); DIRECT_MULTIDIM_ELEM(N2n, n) = abs(DIRECT_MULTIDIM_ELEM(FFT_In, n)); DIRECT_MULTIDIM_ELEM(N2n, n) *= DIRECT_MULTIDIM_ELEM(N2n, n); } #ifdef DEBUG save() = S2s(); save.write("PPPS2s.xmp"); save() = N2s(); save.write("PPPN2s.xmp"); save() = S2n(); save.write("PPPS2n.xmp"); save() = N2n(); save.write("PPPN2n.xmp"); #endif if (dim == 2) { // Compute the SSNR image Image<double> SSNR2D; SSNR2D().initZeros(S2s); const MultidimArray<double> & SSNR2Dmatrix=SSNR2D(); FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(S2s) { double ISSNR = 0, alpha = 0, SSNR = 0; double aux = DIRECT_MULTIDIM_ELEM(N2s,n); if (aux > min_power) ISSNR = DIRECT_MULTIDIM_ELEM(S2s,n) / aux; aux = DIRECT_MULTIDIM_ELEM(N2n,n); if (aux > min_power) alpha = DIRECT_MULTIDIM_ELEM(S2n,n) / aux; if (alpha > min_power) { aux = ISSNR / alpha - 1.0; SSNR = XMIPP_MAX(aux, 0.0); } if (SSNR > min_power) DIRECT_MULTIDIM_ELEM(SSNR2Dmatrix,n) = 10.0 * log10(SSNR + 1.0); } CenterFFT(SSNR2D(), true); #ifdef DEBUG save() = SSNR2Dmatrix; save.write("PPPSSNR2D.xmp"); #endif // Save image FileName fn_img_out; fn_img_out.compose(imgno, fn_out_images, "stk"); SSNR2D.write(fn_img_out); size_t objId = SF_individual.addObject(); SF_individual.setValue(MDL_IMAGE,fn_img_out,objId); SF_individual.setValue(MDL_ANGLE_ROT,rot,objId); SF_individual.setValue(MDL_ANGLE_TILT,tilt,objId); SF_individual.setValue(MDL_ANGLE_PSI,psi,objId); }