void Preprocessing::read(int argc, char **argv, int rank)
{
parser.setCommandLine(argc, argv);
int gen_section = parser.addSection("General options");
fn_out = parser.getOption("--o", "Output rootname", "particles");
// Dont allow for directories here!
fn_out = fn_out.removeDirectories();
int particle_section = parser.addSection("Particle selection");
fns_coords_in = parser.getOption("--coord_files", "The coordinate files for all particles to be output (may contain wildcards e.g. \"mics/*.box\" or \"mics/???.star\" )","");
fn_star_in = parser.getOption("--mic_star", "The STAR file with all (selected) micrographs to extract particles from","");
fn_pick_suffix = parser.getOption("--coord_suffix", "The suffix for the coordinate files, e.g. \"_picked.star\" or \".box\"","");
int extract_section = parser.addSection("Particle extraction");
do_extract = parser.checkOption("--extract", "Extract all particles from the micrographs");
extract_size = textToInteger(parser.getOption("--extract_size", "Size of the box to extract the particles in (in pixels)", "-1"));
extract_bias_x = textToInteger(parser.getOption("--extract_bias_x", "Bias in X-direction of picked particles (this value in pixels will be added to the coords)", "0"));
extract_bias_y = textToInteger(parser.getOption("--extract_bias_y", "Bias in Y-direction of picked particles (this value in pixels will be added to the coords)", "0"));
do_movie_extract = parser.checkOption("--extract_movies", "Extract particles from movie stacks (e.g. from DDDs)");
avg_n_frames = textToInteger(parser.getOption("--avg_movie_frames", "Average over this number of individual movie frames", "1"));
movie_first_frame = textToInteger(parser.getOption("--first_movie_frame", "Extract from this movie frame onwards", "1"));
movie_first_frame--; // (start counting at 0, not 1)
movie_last_frame = textToInteger(parser.getOption("--last_movie_frame", "Extract until this movie frame (default=all movie frames)", "0"));
movie_last_frame--; // (start counting at 0, not 1)
fn_movie = parser.getOption("--movie_rootname", "Common name to relate movies to the single micrographs (e.g. mic001_movie.mrcs related to mic001.mrc)", "movie");
int perpart_section = parser.addSection("Particle operations");
do_project_3d = parser.checkOption("--project3d", "Project sub-tomograms along Z to generate 2D particles");
scale = textToInteger(parser.getOption("--scale", "Re-scale the particles to this size (in pixels)", "-1"));
window = textToInteger(parser.getOption("--window", "Re-window the particles to this size (in pixels)", "-1"));
do_normalise = parser.checkOption("--norm", "Normalise the background to average zero and stddev one");
bg_radius = textToInteger(parser.getOption("--bg_radius", "Radius of the circular mask that will be used to define the background area (in pixels)", "-1"));
white_dust_stddev = textToFloat(parser.getOption("--white_dust", "Sigma-values above which white dust will be removed (negative value means no dust removal)","-1"));
black_dust_stddev = textToFloat(parser.getOption("--black_dust", "Sigma-values above which black dust will be removed (negative value means no dust removal)","-1"));
do_invert_contrast = parser.checkOption("--invert_contrast", "Invert the contrast in the input images");
fn_operate_in = parser.getOption("--operate_on", "The operations above are applied to all extracted particles. Use this option to operate on an input stack/STAR file", "");
// Initialise verb for non-parallel execution
verb = 1;
if (!( checkParameter(argc, argv, "--o") || checkParameter(argc, argv, "--operate_on") ))
REPORT_ERROR("Provide either --o or --operate_on");
// Check for errors in the command-line option
if (parser.checkForErrors())
REPORT_ERROR("Errors encountered on the command line (see above), exiting...");
}
//---------------------------------------------------------------------------
int32_t CSVFile::readFloat(uint32_t row, uint32_t col, float& value)
{
int32_t result = seekRowCol(row, col);
if(result == NO_ERROR)
{
value = textToFloat(dataBuffer);
}
else
value = 0.0f;
return(NO_ERROR);
}
void ProgXrayImport::readCorrectionInfo(const FileName &fn, double ¤tBeam,
double &expTime, double &slitWidth) const
{
FileName fnBase = fn.withoutExtension();
std::ifstream fhPosition;
fhPosition.open((fnBase+"-positions.txt").c_str());
if (!fhPosition)
REPORT_ERROR(ERR_IO_NOTEXIST,fnBase+"-positions.txt");
int itemsFound = 0;
std::string line;
std::vector<std::string> tokens;
while (!fhPosition.eof())
{
getline(fhPosition,line);
splitString(line," ",tokens);
if (tokens[0]=="sr:current")
{
currentBeam = textToFloat(tokens[1]);
itemsFound++;
}
else if (tokens[0]=="xm:ccd:exp_time")
{
expTime = textToFloat(tokens[1]);
itemsFound++;
}
else if (tokens[0]=="xm:mono:slitwidth")
{
slitWidth = textToFloat(tokens[1]);
itemsFound++;
}
if (itemsFound==3)
break;
}
fhPosition.close();
if ( itemsFound != 3 )
REPORT_ERROR(ERR_VALUE_EMPTY,(std::string)"Cannot find all parameters in "+
fnBase+"-positions.txt");
}
//---------------------------------------------------------------------------
long CSVFile::readFloat (DWORD row, DWORD col, float &value)
{
long result = seekRowCol(row,col);
if (result == NO_ERR)
{
value = textToFloat(dataBuffer);
}
else
value = 0.0f;
return(NO_ERR);
}
void read(int argc, char **argv)
{
parser.setCommandLine(argc, argv);
int general_section = parser.addSection("General options");
fn_unt = parser.getOption("--u", "Input STAR file with untilted particles");
fn_til = parser.getOption("--t", "Input STAR file with tilted particles");
fn_eps = parser.getOption("--o", "Output EPS file ", "tiltpair.eps");
fn_sym = parser.getOption("--sym", "Symmetry point group", "C1");
exp_tilt = textToFloat(parser.getOption("--exp_tilt", "Choose symmetry operator that gives tilt angle closest to this value", "0."));
exp_beta = textToFloat(parser.getOption("--exp_beta", "Choose symmetry operator that gives beta angle closest to this value", "0."));
dist_from_alpha = textToFloat(parser.getOption("--dist_from_alpha", "Direction (alpha angle) of tilt axis from which to calculate distance", "0."));
dist_from_tilt = textToFloat(parser.getOption("--dist_from_tilt", "Tilt angle from which to calculate distance", "0."));
plot_max_tilt = textToFloat(parser.getOption("--max_tilt", "Maximum tilt angle to plot in the EPS file", "90."));
plot_spot_radius = textToInteger(parser.getOption("--spot_radius", "Radius in pixels of the spots in the tiltpair plot", "3"));
// Check for errors in the command-line option
if (parser.checkForErrors())
REPORT_ERROR("Errors encountered on the command line, exiting...");
}
void Postprocessing::read(int argc, char **argv)
{
parser.setCommandLine(argc, argv);
int gen_section = parser.addSection("General options");
fn_in = parser.getOption("--i", "Input rootname, e.g. run1");
fn_out = parser.getOption("--o", "Output rootname", "postprocess");
angpix = textToFloat(parser.getOption("--angpix", "Pixel size in Angstroms"));
int mask_section = parser.addSection("Masking options");
do_auto_mask = parser.checkOption("--auto_mask", "Perform automated masking, based on a density threshold");
ini_mask_density_threshold = textToFloat(parser.getOption("--inimask_threshold", "Density at which to threshold the map for the initial seed mask", "0.02"));
extend_ini_mask = textToFloat(parser.getOption("--extend_inimask", "Number of pixels to extend the initial seed mask", "3."));
width_soft_mask_edge = textToFloat(parser.getOption("--width_mask_edge", "Width for the raised cosine soft mask edge (in pixels)", "6."));
fn_mask = parser.getOption("--mask", "Filename of a user-provided mask (1=protein, 0=solvent, all values in range [0,1])", "");
int sharp_section = parser.addSection("Sharpening options");
fn_mtf = parser.getOption("--mtf", "User-provided STAR-file with the MTF-curve of the detector", "");
do_auto_bfac = parser.checkOption("--auto_bfac", "Perform automated B-factor determination (Rosenthal and Henderson, 2003)");
fit_minres = textToFloat(parser.getOption("--autob_lowres", "Lowest resolution (in A) to include in fitting of the B-factor", "10."));
fit_maxres = textToFloat(parser.getOption("--autob_highres", "Highest resolution (in A) to include in fitting of the B-factor", "0."));
adhoc_bfac = textToFloat(parser.getOption("--adhoc_bfac", "User-provided B-factor (in A^2) for map sharpening, e.g. -400", "0."));
int filter_section = parser.addSection("Filtering options");
do_fsc_weighting = !parser.checkOption("--skip_fsc_weighting", "Do not use FSC-weighting (Rosenthal and Henderson, 2003) in the sharpening process");
// include low-pass filter option in the program? This could be useful for structurally heterogeneous reconstructions (instead of FSC-weighting)
low_pass_freq = textToFloat(parser.getOption("--low_pass", "Resolution (in Angstroms) at which to low-pass filter the final map (by default at final resolution)", "0."));
int expert_section = parser.addSection("Expert options");
randomize_fsc_at = textToFloat(parser.getOption("--randomize_at_fsc", "Randomize phases from the resolution where FSC drops below this value", "0.8"));
filter_edge_width = textToInteger(parser.getOption("--filter_edge_width", "Width of the raised cosine on the low-pass filter edge (in resolution shells)", "2"));
verb = textToInteger(parser.getOption("--verb", "Verbosity", "1"));
// Check for errors in the command-line option
if (parser.checkForErrors())
REPORT_ERROR("Errors encountered on the command line (see above), exiting...");
}
void ProgXrayImport::readGeoInfo(const FileName &fn, MDRow &rowGeo) const
{
double tiltAngle;
switch (dSource)
{
case MISTRAL:
tiltAngle = dMi(anglesArray, fn.getPrefixNumber()-1);
break;
case BESSY:
case GENERIC:
{
FileName fnBase = fn.withoutExtension();
std::ifstream fhPosition;
fhPosition.open((fnBase+"-positions.txt").c_str());
if (!fhPosition)
REPORT_ERROR(ERR_IO_NOTEXIST,fnBase+"-positions.txt");
int itemsFound=0;
std::string line;
std::vector<std::string> tokens;
while (!fhPosition.eof())
{
getline(fhPosition,line);
splitString(line," ",tokens);
if (tokens[0]=="xm:sample:rx")
{
tiltAngle=textToFloat(tokens[1]);
itemsFound++;
}
if (itemsFound==1)
break;
}
fhPosition.close();
if (itemsFound!=1)
REPORT_ERROR(ERR_VALUE_EMPTY,(std::string)"Cannot find tilt angle in "+
fnBase+"-positions.txt");
}
break;
}
rowGeo.setValue(MDL_ANGLE_TILT, tiltAngle);
}
void read(int argc, char **argv)
{
parser.setCommandLine(argc, argv);
int general_section = parser.addSection("General options");
fn1 = parser.getOption("--i1", "1st input STAR file ");
fn2 = parser.getOption("--i2", "2nd input STAR file ");
fn_both = parser.getOption("--both", "Output STAR file with entries from both input STAR files ", "");
fn_only1 = parser.getOption("--only1", "Output STAR file with entries that only occur in the 1st input STAR files ", "");
fn_only2 = parser.getOption("--only2", "Output STAR file with entries that only occur in the 2nd input STAR files ", "");
fn_label1 = parser.getOption("--label1", "1st metadata label for the comparison (may be string, int or DOUBLE)", "");
fn_label2 = parser.getOption("--label2", "2nd metadata label for the comparison (DOUBLE only) for 2D/3D-distance)", "");
fn_label3 = parser.getOption("--label3", "3rd metadata label for the comparison (DOUBLE only) for 3D-distance)", "");
eps = textToFloat(parser.getOption("--max_dist", "Maximum distance to consider a match (for int and DOUBLE only)", "0."));
// Check for errors in the command-line option
if (parser.checkForErrors())
REPORT_ERROR("Errors encountered on the command line, exiting...");
}
void read(int argc, char **argv)
{
parser.setCommandLine(argc, argv);
int general_section = parser.addSection("General Options");
fn_unt = parser.getOption("--u", "STAR file with the untilted xy-coordinates");
fn_til = parser.getOption("--t", "STAR file with the untilted xy-coordinates");
size = textToInteger(parser.getOption("--size", "Largest dimension of the micrograph (in pixels), e.g. 4096"));
dim = textToInteger(parser.getOption("--dim", "Dimension of boxed particles (for EMAN .box files in pixels)", "200"));
acc = textToFloat(parser.getOption("--acc", "Allowed accuracy (in pixels), e.g. half the particle diameter"));
tilt = textToFloat(parser.getOption("--tilt", "Fix tilt angle (in degrees)", "99999."));
rot = textToFloat(parser.getOption("--rot", "Fix direction of the tilt axis (in degrees), 0 = along y, 90 = along x", "99999."));
do_opt = !parser.checkOption("--dont_opt", "Skip optimization of the transformation matrix");
mind2 = ROUND(acc * acc);
int angle_section = parser.addSection("Specified tilt axis and translational search ranges");
tilt0 = textToFloat(parser.getOption("--tilt0", "Minimum tilt angle (in degrees)","0."));
tiltF = textToFloat(parser.getOption("--tiltF", "Maximum tilt angle (in degrees)","99999."));
if (tiltF == 99999.) tiltF = tilt0;
tiltStep = textToFloat(parser.getOption("--tiltStep", "Tilt angle step size (in degrees)","1."));
rot0 = textToFloat(parser.getOption("--rot0", "Minimum rot angle (in degrees)","0."));
rotF = textToFloat(parser.getOption("--rotF", "Maximum rot angle (in degrees)","99999."));
if (rotF == 99999.) rotF = rot0;
rotStep = textToFloat(parser.getOption("--rotStep", "Rot angle step size (in degrees)","1."));
x0 = textToInteger(parser.getOption("--x0", "Minimum X offset (pixels)","-99999"));
xF = textToInteger(parser.getOption("--xF", "Maximum X offset (pixels)","99999"));
xStep = textToInteger(parser.getOption("--xStep", "X offset step size (pixels)","-1"));
y0 = textToInteger(parser.getOption("--y0", "Minimum Y offset (pixels)","-99999"));
yF = textToInteger(parser.getOption("--yF", "Maximum Y offset (pixels)","99999"));
yStep = textToInteger(parser.getOption("--yStep", "Y offset step size (pixels)","-1"));
// Check for errors in the command-line option
if (parser.checkForErrors())
REPORT_ERROR("Errors encountered on the command line, exiting...");
// If tilt and rot were given: do not search those
if (tilt != 99999.)
{
tilt0 = tiltF = tilt;
tiltStep = 1.;
}
if (rot != 99999.)
{
rot0 = rotF = rot;
rotStep = 1.;
}
// By default search the entire micrograph
x0 = XMIPP_MAX(x0, -size);
xF = XMIPP_MIN(xF, size);
// By default use a xStep of one third the accuracy
if (xStep < 0)
xStep = acc / 3;
// By default treat y search in the same way as the x-search
if (y0 == -99999)
y0 = x0;
if (yF == 99999)
yF = xF;
if (yStep < 0)
yStep = xStep;
// Done reading, now fill p_unt and p_til
MDunt.read(fn_unt);
MDtil.read(fn_til);
// Check for the correct labels
if (!MDunt.containsLabel(EMDL_IMAGE_COORD_X) || !MDunt.containsLabel(EMDL_IMAGE_COORD_Y))
REPORT_ERROR("ERROR: Untilted STAR file does not contain the rlnCoordinateX or Y labels");
if (!MDtil.containsLabel(EMDL_IMAGE_COORD_X) || !MDtil.containsLabel(EMDL_IMAGE_COORD_Y))
REPORT_ERROR("ERROR: Tilted STAR file does not contain the rlnCoordinateX or Y labels");
double x, y;
p_unt.clear();
FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDunt)
{
MDunt.getValue(EMDL_IMAGE_COORD_X, x);
MDunt.getValue(EMDL_IMAGE_COORD_Y, y);
p_unt.push_back((int)x);
p_unt.push_back((int)y);
}
p_til.clear();
FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDtil)
{
MDtil.getValue(EMDL_IMAGE_COORD_X, x);
MDtil.getValue(EMDL_IMAGE_COORD_Y, y);
p_til.push_back((int)x);
p_til.push_back((int)y);
}
// Initialize best transformation params
best_x = best_y = 9999;
best_rot = best_tilt = 9999.;
}
// Read Symmetry file ======================================================
// crystal symmetry matices from http://cci.lbl.gov/asu_gallery/
int SymList::read_sym_file(FileName fn_sym)
{
int i, j;
FILE *fpoii;
char line[80];
char *auxstr;
DOUBLE ang_incr, rot_ang;
int fold;
Matrix2D<DOUBLE> L(4, 4), R(4, 4);
Matrix1D<DOUBLE> axis(3);
int pgGroup = 0, pgOrder = 0;
std::vector<std::string> fileContent;
//check if reserved word
// Open file ---------------------------------------------------------
if ((fpoii = fopen(fn_sym.c_str(), "r")) == NULL)
{
//check if reserved word and return group and order
if (isSymmetryGroup(fn_sym, pgGroup, pgOrder))
{
fill_symmetry_class(fn_sym, pgGroup, pgOrder, fileContent);
}
else
REPORT_ERROR((std::string)"SymList::read_sym_file:Can't open file: "
+ " or do not recognize symmetry group" + fn_sym);
}
else
{
while (fgets(line, 79, fpoii) != NULL)
{
if (line[0] == ';' || line[0] == '#' || line[0] == '\0')
continue;
fileContent.push_back(line);
}
fclose(fpoii);
}
// Count the number of symmetries ------------------------------------
true_symNo = 0;
// count number of axis and mirror planes. It will help to identify
// the crystallographic symmetry
int no_axis, no_mirror_planes, no_inversion_points;
no_axis = no_mirror_planes = no_inversion_points = 0;
for (int n=0; n<fileContent.size(); n++)
{
strcpy(line,fileContent[n].c_str());
auxstr = firstToken(line);
if (auxstr == NULL)
{
std::cout << line;
std::cout << "Wrong line in symmetry file, the line is skipped\n";
continue;
}
if (strcmp(auxstr, "rot_axis") == 0)
{
auxstr = nextToken();
fold = textToInteger(auxstr);
true_symNo += (fold - 1);
no_axis++;
}
else if (strcmp(auxstr, "mirror_plane") == 0)
{
true_symNo++;
no_mirror_planes++;
}
else if (strcmp(auxstr, "inversion") == 0)
{
true_symNo += 1;
no_inversion_points = 1;
}
}
// Ask for memory
__L.resize(4*true_symNo, 4);
__R.resize(4*true_symNo, 4);
__chain_length.resize(true_symNo);
__chain_length.initConstant(1);
// Read symmetry parameters
i = 0;
for (int n=0; n<fileContent.size(); n++)
{
strcpy(line,fileContent[n].c_str());
auxstr = firstToken(line);
// Rotational axis ---------------------------------------------------
if (strcmp(auxstr, "rot_axis") == 0)
{
auxstr = nextToken();
fold = textToInteger(auxstr);
auxstr = nextToken();
XX(axis) = textToDOUBLE(auxstr);
auxstr = nextToken();
YY(axis) = textToDOUBLE(auxstr);
auxstr = nextToken();
ZZ(axis) = textToDOUBLE(auxstr);
ang_incr = 360. / fold;
L.initIdentity();
for (j = 1, rot_ang = ang_incr; j < fold; j++, rot_ang += ang_incr)
{
rotation3DMatrix(rot_ang, axis, R);
R.setSmallValuesToZero();
set_matrices(i++, L, R.transpose());
}
__sym_elements++;
// inversion ------------------------------------------------------
}
else if (strcmp(auxstr, "inversion") == 0)
{
L.initIdentity();
L(2, 2) = -1;
R.initIdentity();
R(0, 0) = -1.;
R(1, 1) = -1.;
R(2, 2) = -1.;
set_matrices(i++, L, R);
__sym_elements++;
// mirror plane -------------------------------------------------------------
}
else if (strcmp(auxstr, "mirror_plane") == 0)
{
auxstr = nextToken();
XX(axis) = textToFloat(auxstr);
auxstr = nextToken();
YY(axis) = textToFloat(auxstr);
auxstr = nextToken();
ZZ(axis) = textToFloat(auxstr);
L.initIdentity();
L(2, 2) = -1;
Matrix2D<DOUBLE> A;
alignWithZ(axis,A);
A = A.transpose();
R = A * L * A.inv();
L.initIdentity();
set_matrices(i++, L, R);
__sym_elements++;
}
}
compute_subgroup();
return pgGroup;
}
void ProgVolumePCA::run()
{
show();
produce_side_info();
const MultidimArray<int> &imask=mask.imask;
size_t Nvoxels=imask.sum();
MultidimArray<float> v;
v.initZeros(Nvoxels);
// Add all volumes to the analyzer
FileName fnVol;
FOR_ALL_OBJECTS_IN_METADATA(mdVols)
{
mdVols.getValue(MDL_IMAGE,fnVol,__iter.objId);
V.read(fnVol);
// Construct vector
const MultidimArray<double> &mV=V();
size_t idx=0;
FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(mV)
{
if (DIRECT_MULTIDIM_ELEM(imask,n))
DIRECT_MULTIDIM_ELEM(v,idx++)=DIRECT_MULTIDIM_ELEM(mV,n);
}
analyzer.addVector(v);
}
// Construct PCA basis
analyzer.subtractAvg();
analyzer.learnPCABasis(NPCA,100);
// Project onto the PCA basis
Matrix2D<double> proj;
analyzer.projectOnPCABasis(proj);
std::vector<double> dimredProj;
dimredProj.resize(NPCA);
int i=0;
FOR_ALL_OBJECTS_IN_METADATA(mdVols)
{
memcpy(&dimredProj[0],&MAT_ELEM(proj,i,0),NPCA*sizeof(double));
mdVols.setValue(MDL_DIMRED,dimredProj,__iter.objId);
i++;
}
if (fnVolsOut!="")
mdVols.write(fnVolsOut);
else
mdVols.write(fnVols);
// Save the basis
const MultidimArray<double> &mV=V();
for (int i=NPCA-1; i>=0; --i)
{
V().initZeros();
size_t idx=0;
const MultidimArray<double> &mPCA=analyzer.PCAbasis[i];
FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(mV)
{
if (DIRECT_MULTIDIM_ELEM(imask,n))
DIRECT_MULTIDIM_ELEM(mV,n)=DIRECT_MULTIDIM_ELEM(mPCA,idx++);
}
if (fnBasis!="")
V.write(fnBasis,i+1,true,WRITE_OVERWRITE);
}
// Generate the PCA volumes
if (listOfPercentiles.size()>0 && fnOutStack!="" && fnAvgVol!="")
{
Image<double> Vavg;
if (fnAvgVol!="")
Vavg.read(fnAvgVol);
else
Vavg().initZeros(V());
Matrix1D<double> p;
proj.toVector(p);
Matrix1D<double> psorted=p.sort();
Image<double> Vpca;
Vpca()=Vavg();
createEmptyFile(fnOutStack,(int)XSIZE(Vavg()),(int)YSIZE(Vavg()),(int)ZSIZE(Vavg()),listOfPercentiles.size());
std::cout << "listOfPercentiles.size()=" << listOfPercentiles.size() << std::endl;
for (size_t i=0; i<listOfPercentiles.size(); i++)
{
int idx=(int)round(textToFloat(listOfPercentiles[i].c_str())/100.0*VEC_XSIZE(p));
std::cout << "Percentile " << listOfPercentiles[i] << " -> idx=" << idx << " p(idx)=" << psorted(idx) << std::endl;
Vpca()+=psorted(idx)*V();
Vpca.write(fnOutStack,i+1,true,WRITE_REPLACE);
}
}
}
