void Preprocessing::initialise()
{
if (fns_coords_in == "" && fn_star_in == "" && fn_operate_in == "" && !do_join_starfile)
REPORT_ERROR("Provide either --extract, --join_starfile or --operate_on");
// Set up which coordinate files to extract particles from (or to join STAR file for)
do_join_starfile = false;
if (fns_coords_in != "" || fn_star_in != "")
{
do_join_starfile = true;
if (do_extract && verb > 0)
{
if (extract_size < 0)
REPORT_ERROR("Preprocessing::initialise ERROR: please provide the size of the box to extract particle using --extract_size ");
std::cout << " Extract particles based on the following coordinate files: " << std::endl;
}
else if (!do_extract && verb > 0)
{
std::cout << " Creating output STAR file for particles based on the following coordinate files: " << std::endl;
}
// Get the filenames of all micrographs to be processed by CTFFIND
if (fns_coords_in != "")
{
fns_coords_in.globFiles(fn_coords);
}
else if (fn_star_in != "")
{
MetaDataTable MDmics;
MDmics.read(fn_star_in);
if (!MDmics.containsLabel(EMDL_MICROGRAPH_NAME))
REPORT_ERROR("Preprocessing::initialise ERROR: Input micrograph STAR file has no rlnMicrographName column!");
fn_coords.clear();
FileName fn_mic;
FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDmics)
{
MDmics.getValue(EMDL_MICROGRAPH_NAME, fn_mic);
fn_mic = fn_mic.withoutExtension() + fn_pick_suffix;
if (exists(fn_mic))
fn_coords.push_back(fn_mic);
else if (verb > 0)
std::cout << "Warning: coordinate file " << fn_mic << " does not exist..." << std::endl;
}
}
void compareMetaDataTable(MetaDataTable &MD1, MetaDataTable &MD2,
MetaDataTable &MDboth, MetaDataTable &MDonly1, MetaDataTable &MDonly2,
EMDLabel label1, DOUBLE eps, EMDLabel label2, EMDLabel label3)
{
if (!MD1.containsLabel(label1))
REPORT_ERROR("compareMetaDataTableEqualLabel::ERROR MD1 does not contain the specified label1.");
if (!MD2.containsLabel(label1))
REPORT_ERROR("compareMetaDataTableEqualLabel::ERROR MD2 does not contain the specified label1.");
if (label2 != EMDL_UNDEFINED)
{
if (!EMDL::isDouble(label1) || !EMDL::isDouble(label2))
REPORT_ERROR("compareMetaDataTableEqualLabel::ERROR 2D or 3D distances are only allowed for DOUBLEs.");
if (!MD1.containsLabel(label2))
REPORT_ERROR("compareMetaDataTableEqualLabel::ERROR MD1 does not contain the specified label2.");
if (!MD2.containsLabel(label2))
REPORT_ERROR("compareMetaDataTableEqualLabel::ERROR MD2 does not contain the specified label2.");
}
if (label3 != EMDL_UNDEFINED)
{
if (!EMDL::isDouble(label3))
REPORT_ERROR("compareMetaDataTableEqualLabel::ERROR 3D distances are only allowed for DOUBLEs.");
if (!MD1.containsLabel(label3))
REPORT_ERROR("compareMetaDataTableEqualLabel::ERROR MD1 does not contain the specified label3.");
if (!MD2.containsLabel(label3))
REPORT_ERROR("compareMetaDataTableEqualLabel::ERROR MD2 does not contain the specified label3.");
}
MDboth.clear();
MDonly1.clear();
MDonly2.clear();
std::string mystr1, mystr2;
int myint1, myint2;
DOUBLE myd1, myd2, mydy1 = 0., mydy2 = 0., mydz1 = 0., mydz2 = 0.;
// loop over MD1
std::vector<long int> to_remove_from_only2;
for (long int current_object1 = MD1.firstObject();
current_object1 != MetaDataTable::NO_MORE_OBJECTS && current_object1 != MetaDataTable::NO_OBJECTS_STORED;
current_object1 = MD1.nextObject())
{
if (EMDL::isString(label1))
MD1.getValue(label1, mystr1);
else if (EMDL::isInt(label1))
MD1.getValue(label1, myint1);
else if (EMDL::isDouble(label1))
{
MD1.getValue(label1, myd1);
if (label2 != EMDL_UNDEFINED)
MD1.getValue(label2, mydy1);
if (label3 != EMDL_UNDEFINED)
MD1.getValue(label3, mydz1);
}
else
REPORT_ERROR("compareMetaDataTableEqualLabel ERROR: only implemented for strings, integers or DOUBLEs");
// loop over MD2
bool have_in_2 = false;
for (long int current_object2 = MD2.firstObject();
current_object2 != MetaDataTable::NO_MORE_OBJECTS && current_object2 != MetaDataTable::NO_OBJECTS_STORED;
current_object2 = MD2.nextObject())
{
if (EMDL::isString(label1))
{
MD2.getValue(label1, mystr2);
if (strcmp(mystr1.c_str(), mystr2.c_str()) == 0)
{
have_in_2 = true;
to_remove_from_only2.push_back(current_object2);
MDboth.addObject(MD1.getObject());
break;
}
}
else if (EMDL::isInt(label1))
{
MD2.getValue(label1, myint2);
if ( ABS(myint2 - myint1) <= ROUND(eps) )
{
have_in_2 = true;
to_remove_from_only2.push_back(current_object2);
MDboth.addObject(MD1.getObject());
break;
}
}
else if (EMDL::isDouble(label1))
{
MD2.getValue(label1, myd2);
if (label2 != EMDL_UNDEFINED)
MD2.getValue(label2, mydy2);
if (label3 != EMDL_UNDEFINED)
MD2.getValue(label3, mydz2);
DOUBLE dist = sqrt( (myd1 - myd2) * (myd1 - myd2) +
(mydy1 - mydy2) * (mydy1 - mydy2) +
(mydz1 - mydz2) * (mydz1 - mydz2) );
if ( ABS(dist) <= eps )
{
have_in_2 = true;
to_remove_from_only2.push_back(current_object2);
//std::cerr << " current_object1= " << current_object1 << std::endl;
//std::cerr << " myd1= " << myd1 << " myd2= " << myd2 << " mydy1= " << mydy1 << " mydy2= " << mydy2 << " dist= "<<dist<<std::endl;
//std::cerr << " to be removed current_object2= " << current_object2 << std::endl;
MDboth.addObject(MD1.getObject());
break;
}
}
}
if (!have_in_2)
{
MDonly1.addObject(MD1.getObject());
}
}
for (long int current_object2 = MD2.firstObject();
current_object2 != MetaDataTable::NO_MORE_OBJECTS && current_object2 != MetaDataTable::NO_OBJECTS_STORED;
current_object2 = MD2.nextObject())
{
bool to_be_removed = false;
for (long int i = 0; i < to_remove_from_only2.size(); i++)
{
if (to_remove_from_only2[i] == current_object2)
{
to_be_removed = true;
break;
}
}
if (!to_be_removed)
{
//std::cerr << " doNOT remove current_object2= " << current_object2 << std::endl;
MDonly2.addObject(MD2.getObject(current_object2));
}
}
}
void read(int argc, char **argv)
{
parser.setCommandLine(argc, argv);
int general_section = parser.addSection("General Options");
fn_unt = parser.getOption("--u", "STAR file with the untilted xy-coordinates");
fn_til = parser.getOption("--t", "STAR file with the untilted xy-coordinates");
size = textToInteger(parser.getOption("--size", "Largest dimension of the micrograph (in pixels), e.g. 4096"));
dim = textToInteger(parser.getOption("--dim", "Dimension of boxed particles (for EMAN .box files in pixels)", "200"));
acc = textToFloat(parser.getOption("--acc", "Allowed accuracy (in pixels), e.g. half the particle diameter"));
tilt = textToFloat(parser.getOption("--tilt", "Fix tilt angle (in degrees)", "99999."));
rot = textToFloat(parser.getOption("--rot", "Fix direction of the tilt axis (in degrees), 0 = along y, 90 = along x", "99999."));
do_opt = !parser.checkOption("--dont_opt", "Skip optimization of the transformation matrix");
mind2 = ROUND(acc * acc);
int angle_section = parser.addSection("Specified tilt axis and translational search ranges");
tilt0 = textToFloat(parser.getOption("--tilt0", "Minimum tilt angle (in degrees)","0."));
tiltF = textToFloat(parser.getOption("--tiltF", "Maximum tilt angle (in degrees)","99999."));
if (tiltF == 99999.) tiltF = tilt0;
tiltStep = textToFloat(parser.getOption("--tiltStep", "Tilt angle step size (in degrees)","1."));
rot0 = textToFloat(parser.getOption("--rot0", "Minimum rot angle (in degrees)","0."));
rotF = textToFloat(parser.getOption("--rotF", "Maximum rot angle (in degrees)","99999."));
if (rotF == 99999.) rotF = rot0;
rotStep = textToFloat(parser.getOption("--rotStep", "Rot angle step size (in degrees)","1."));
x0 = textToInteger(parser.getOption("--x0", "Minimum X offset (pixels)","-99999"));
xF = textToInteger(parser.getOption("--xF", "Maximum X offset (pixels)","99999"));
xStep = textToInteger(parser.getOption("--xStep", "X offset step size (pixels)","-1"));
y0 = textToInteger(parser.getOption("--y0", "Minimum Y offset (pixels)","-99999"));
yF = textToInteger(parser.getOption("--yF", "Maximum Y offset (pixels)","99999"));
yStep = textToInteger(parser.getOption("--yStep", "Y offset step size (pixels)","-1"));
// Check for errors in the command-line option
if (parser.checkForErrors())
REPORT_ERROR("Errors encountered on the command line, exiting...");
// If tilt and rot were given: do not search those
if (tilt != 99999.)
{
tilt0 = tiltF = tilt;
tiltStep = 1.;
}
if (rot != 99999.)
{
rot0 = rotF = rot;
rotStep = 1.;
}
// By default search the entire micrograph
x0 = XMIPP_MAX(x0, -size);
xF = XMIPP_MIN(xF, size);
// By default use a xStep of one third the accuracy
if (xStep < 0)
xStep = acc / 3;
// By default treat y search in the same way as the x-search
if (y0 == -99999)
y0 = x0;
if (yF == 99999)
yF = xF;
if (yStep < 0)
yStep = xStep;
// Done reading, now fill p_unt and p_til
MDunt.read(fn_unt);
MDtil.read(fn_til);
// Check for the correct labels
if (!MDunt.containsLabel(EMDL_IMAGE_COORD_X) || !MDunt.containsLabel(EMDL_IMAGE_COORD_Y))
REPORT_ERROR("ERROR: Untilted STAR file does not contain the rlnCoordinateX or Y labels");
if (!MDtil.containsLabel(EMDL_IMAGE_COORD_X) || !MDtil.containsLabel(EMDL_IMAGE_COORD_Y))
REPORT_ERROR("ERROR: Tilted STAR file does not contain the rlnCoordinateX or Y labels");
double x, y;
p_unt.clear();
FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDunt)
{
MDunt.getValue(EMDL_IMAGE_COORD_X, x);
MDunt.getValue(EMDL_IMAGE_COORD_Y, y);
p_unt.push_back((int)x);
p_unt.push_back((int)y);
}
p_til.clear();
FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDtil)
{
MDtil.getValue(EMDL_IMAGE_COORD_X, x);
MDtil.getValue(EMDL_IMAGE_COORD_Y, y);
p_til.push_back((int)x);
p_til.push_back((int)y);
}
// Initialize best transformation params
best_x = best_y = 9999;
best_rot = best_tilt = 9999.;
}
void run()
{
MD.read(fn_star);
// Check for rlnImageName label
if (!MD.containsLabel(EMDL_IMAGE_NAME))
REPORT_ERROR("ERROR: Input STAR file does not contain the rlnImageName label");
if (do_split_per_micrograph && !MD.containsLabel(EMDL_MICROGRAPH_NAME))
REPORT_ERROR("ERROR: Input STAR file does not contain the rlnMicrographName label");
Image<DOUBLE> in;
FileName fn_img, fn_mic;
std::vector<FileName> fn_mics;
std::vector<int> mics_ndims;
// First get number of images and their size
int ndim=0;
bool is_first=true;
int xdim, ydim, zdim;
FOR_ALL_OBJECTS_IN_METADATA_TABLE(MD)
{
if (is_first)
{
MD.getValue(EMDL_IMAGE_NAME, fn_img);
in.read(fn_img);
xdim=XSIZE(in());
ydim=YSIZE(in());
zdim=ZSIZE(in());
is_first=false;
}
if (do_split_per_micrograph)
{
MD.getValue(EMDL_MICROGRAPH_NAME, fn_mic);
bool have_found = false;
for (int m = 0; m < fn_mics.size(); m++)
{
if (fn_mic == fn_mics[m])
{
have_found = true;
mics_ndims[m]++;
break;
}
}
if (!have_found)
{
fn_mics.push_back(fn_mic);
mics_ndims.push_back(1);
}
}
ndim++;
}
// If not splitting, just fill fn_mics and mics_ndim with one entry (to re-use loop below)
if (!do_split_per_micrograph)
{
fn_mics.push_back("");
mics_ndims.push_back(ndim);
}
// Loop over all micrographs
for (int m = 0; m < fn_mics.size(); m++)
{
ndim = mics_ndims[m];
fn_mic = fn_mics[m];
// Resize the output image
std::cout << "Resizing the output stack to "<< ndim<<" images of size: "<<xdim<<"x"<<ydim<<"x"<<zdim << std::endl;
DOUBLE Gb = ndim*zdim*ydim*xdim*8./1024./1024./1024.;
std::cout << "This will require " << Gb << "Gb of memory...."<< std::endl;
Image<DOUBLE> out(xdim, ydim, zdim, ndim);
int n = 0;
init_progress_bar(ndim);
FOR_ALL_OBJECTS_IN_METADATA_TABLE(MD)
{
FileName fn_mymic;
if (do_split_per_micrograph)
MD.getValue(EMDL_MICROGRAPH_NAME, fn_mymic);
else
fn_mymic="";
if (fn_mymic == fn_mic)
{
MD.getValue(EMDL_IMAGE_NAME, fn_img);
in.read(fn_img);
if (do_apply_trans)
{
DOUBLE xoff = 0.;
DOUBLE yoff = 0.;
DOUBLE psi = 0.;
MD.getValue(EMDL_ORIENT_ORIGIN_X, xoff);
MD.getValue(EMDL_ORIENT_ORIGIN_Y, yoff);
MD.getValue(EMDL_ORIENT_PSI, psi);
// Apply the actual transformation
Matrix2D<DOUBLE> A;
rotation2DMatrix(psi, A);
MAT_ELEM(A,0, 2) = xoff;
MAT_ELEM(A,1, 2) = yoff;
selfApplyGeometry(in(), A, IS_NOT_INV, DONT_WRAP);
}
out().setImage(n, in());
n++;
if (n%100==0) progress_bar(n);
}
}
progress_bar(ndim);
FileName fn_out;
if (do_split_per_micrograph)
{
// Remove any extensions from micrograph names....
fn_out = fn_root + "_" + fn_mic.withoutExtension() + fn_ext;
}
else
fn_out = fn_root + fn_ext;
out.write(fn_out);
std::cout << "Written out: " << fn_out << std::endl;
}
std::cout << "Done!" <<std::endl;
}
