PTOptEstimator(PanoramaData & pano, int i1, int i2, double maxError,
bool optHFOV, bool optB)
{
m_maxError = maxError;
UIntSet imgs;
imgs.insert(i1);
imgs.insert(i2);
m_localPano = (pano.getNewSubset(imgs)); // don't forget to delete
m_li1 = (i1 < i2) ? 0 : 1;
m_li2 = (i1 < i2) ? 1 : 0;
// get control points
m_cps = m_localPano->getCtrlPoints();
// only use 2D control points
BOOST_FOREACH(ControlPoint & kp, m_cps) {
if (kp.mode == ControlPoint::X_Y) {
m_xy_cps.push_back(kp);
}
}
if (optHFOV)
m_optvars.push_back(OptVarSpec(0,std::string("v")));
if (optB)
m_optvars.push_back(OptVarSpec(0,std::string("b")));
m_optvars.push_back(OptVarSpec(m_li2,"r"));
m_optvars.push_back(OptVarSpec(m_li2,"p"));
m_optvars.push_back(OptVarSpec(m_li2,"y"));
if (optHFOV)
m_optvars.push_back(OptVarSpec(0,"v"));
if (optB)
m_optvars.push_back(OptVarSpec(0,"b"));
/** optimisation for first pass */
m_opt_first_pass.resize(2);
m_opt_first_pass[1].insert("r");
m_opt_first_pass[1].insert("p");
m_opt_first_pass[1].insert("y");
/** optimisation for second pass */
if (optHFOV || optB) {
m_opt_second_pass = m_opt_first_pass;
if (optHFOV)
m_opt_second_pass[0].insert("v");
if (optB)
m_opt_second_pass[0].insert("b");
}
// number of points required for estimation
m_numForEstimate = (m_optvars.size()+1)/2;
// extract initial parameters from pano
m_initParams.resize(m_optvars.size());
int i=0;
BOOST_FOREACH(OptVarSpec & v, m_optvars) {
m_initParams[i] = v.get(*m_localPano);
DEBUG_DEBUG("get init var: " << v.m_name << ", " << v.m_img << ": " << m_initParams[i]);
i++;
}
void RotatePanorama::rotatePano(PanoramaData& panorama, const Matrix3& transformMat)
{
for (unsigned int i = 0; i < panorama.getNrOfImages(); i++)
{
const SrcPanoImage & image = panorama.getImage(i);
SrcPanoImage copy = image;
double y = image.getYaw();
double p = image.getPitch();
double r = image.getRoll();
Matrix3 mat;
mat.SetRotationPT(DEG_TO_RAD(y), DEG_TO_RAD(p), DEG_TO_RAD(r));
DEBUG_DEBUG("rotation matrix (PT) for img " << i << " << ypr:" << y << " " << p << " " << r << std::endl << mat);
Matrix3 rotated;
rotated = transformMat * mat;
DEBUG_DEBUG("rotation matrix after transform: " << rotated);
rotated.GetRotationPT(y,p,r);
y = RAD_TO_DEG(y);
p = RAD_TO_DEG(p);
r = RAD_TO_DEG(r);
DEBUG_DEBUG("rotated angles of img " << i << ": " << y << " " << p << " " << r);
// Don't update a variable linked to a variable we already updated.
conditional_set(Yaw, y);
conditional_set(Pitch, p);
conditional_set(Roll, r);
if(image.getX()!=0.0 || image.getY()!=0.0 || image.getZ()!=0.0)
{
// rotate translation vector
Vector3 vecRot=transformMat.Inverse().TransformVector(Vector3(image.getZ(), image.getX(), image.getY()));
conditional_set(X, vecRot.y);
conditional_set(Y, vecRot.z);
conditional_set(Z, vecRot.x);
// rotate translation plane
mat.SetRotationPT(DEG_TO_RAD(image.getTranslationPlaneYaw()), DEG_TO_RAD(image.getTranslationPlanePitch()), 0.0);
rotated = transformMat * mat;
rotated.GetRotationPT(y,p,r);
conditional_set(TranslationPlaneYaw, RAD_TO_DEG(y));
conditional_set(TranslationPlanePitch, RAD_TO_DEG(p));
};
panorama.setImage(i, copy);
panorama.imageChanged(i);
}
}
unsigned int optimize(PanoramaData& pano,
const char * userScript)
{
char * script = 0;
unsigned int retval = 0;
if (userScript == 0) {
std::ostringstream scriptbuf;
UIntSet allImg;
fill_set(allImg,0, unsigned(pano.getNrOfImages()-1));
pano.printPanoramaScript(scriptbuf, pano.getOptimizeVector(), pano.getOptions(), allImg, true);
script = strdup(scriptbuf.str().c_str());
} else {
script = const_cast<char *>(userScript);
}
OptInfo opt;
AlignInfo ainf;
if (ParseScript( script, &ainf ) == 0)
{
if( CheckParams( &ainf ) == 0 )
{
ainf.fcn = fcnPano;
SetGlobalPtr( &ainf );
opt.numVars = ainf.numParam;
opt.numData = ainf.numPts;
opt.SetVarsToX = SetLMParams;
opt.SetXToVars = SetAlignParams;
opt.fcn = ainf.fcn;
*opt.message = 0;
RunLMOptimizer( &opt );
ainf.data = opt.message;
// get results from align info.
#ifdef DEBUG_WRITE_OPTIM_OUTPUT
fullPath path;
StringtoFullPath(&path, DEBUG_WRITE_OPTIM_OUTPUT_FILE );
ainf.data = opt.message;
WriteResults( script, &path, &ainf, distSquared, 0);
#endif
pano.updateVariables( GetAlignInfoVariables(ainf) );
pano.updateCtrlPointErrors( GetAlignInfoCtrlPoints(ainf) );
} else {
std::cerr << "Bad params" << std::endl;
retval = 2;
}
DisposeAlignInfo( &ainf );
} else {
std::cerr << "Bad params" << std::endl;
retval = 1;
}
if (! userScript) {
free(script);
}
return retval;
}
UIntSet getImagesinROI (const PanoramaData& pano, const UIntSet activeImages)
{
UIntSet images;
PanoramaOptions opts = pano.getOptions();
for (UIntSet::const_iterator it = activeImages.begin(); it != activeImages.end(); ++it)
{
vigra::Rect2D roi = estimateOutputROI(pano, opts, *it);
if (! (roi.isEmpty())) {
images.insert(*it);
}
}
return images;
}
void SmartPhotometricOptimizer::smartOptimizePhotometric(PanoramaData & pano, PhotometricOptimizeMode mode,
const std::vector<vigra_ext::PointPairRGB> & correspondences,
AppBase::ProgressReporter & progress,
double & error)
{
std::vector<SrcPanoImage> ret;
PanoramaOptions opts = pano.getOptions();
if (mode == OPT_PHOTOMETRIC_LDR || mode == OPT_PHOTOMETRIC_LDR_WB) {
// optimize exposure
int vars = OPT_EXP;
optimizePhotometric(pano,
createOptVars(pano, vars, opts.colorReferenceImage),
correspondences, progress, error);
// optimize vignetting & exposure
vars = OPT_EXP | OPT_VIG;
optimizePhotometric(pano,
createOptVars(pano, vars, opts.colorReferenceImage),
correspondences, progress, error);
if (mode == OPT_PHOTOMETRIC_LDR_WB) {
// optimize vignetting & exposure & wb & response
vars = OPT_EXP | OPT_VIG | OPT_RESP | OPT_WB;
} else {
vars = OPT_EXP | OPT_VIG | OPT_RESP;
}
// optimize vignetting & exposure & wb & response
optimizePhotometric(pano,
createOptVars(pano, vars, opts.colorReferenceImage),
correspondences, progress, error);
} else if (mode == OPT_PHOTOMETRIC_HDR || mode == OPT_PHOTOMETRIC_HDR_WB) {
// optimize vignetting
int vars = OPT_VIG;
optimizePhotometric(pano,
createOptVars(pano, vars, opts.colorReferenceImage),
correspondences, progress, error);
// optimize vignetting, wb and response
if (mode == OPT_PHOTOMETRIC_HDR_WB) {
vars = OPT_VIG | OPT_RESP | OPT_WB;
} else {
vars = OPT_VIG | OPT_RESP;
}
optimizePhotometric(pano,
createOptVars(pano, vars, opts.colorReferenceImage),
correspondences, progress, error);
} else {
assert(0 && "Unknown photometric optimisation mode");
}
// adjust
}
vector<UIntSet> getExposureLayers(const PanoramaData & pano, UIntSet allImgs, PanoramaOptions opts)
{
vector<UIntSet> result;
// if no images are available, return empty result vector
if ( allImgs.empty() )
{
return result;
}
UIntSet stack;
do {
unsigned srcImg = *(allImgs.begin());
stack.insert(srcImg);
allImgs.erase(srcImg);
// find all images that have a suitable overlap.
SrcPanoImage simg = pano.getSrcImage(srcImg);
double maxEVDiff = opts.outputLayersExposureDiff;
for (UIntSet::iterator it = allImgs.begin(); it != allImgs.end(); ) {
unsigned srcImg2 = *it;
++it;
SrcPanoImage simg2 = pano.getSrcImage(srcImg2);
if ( fabs(simg.getExposureValue() - simg2.getExposureValue()) < maxEVDiff )
{
stack.insert(srcImg2);
allImgs.erase(srcImg2);
}
}
result.push_back(stack);
stack.clear();
} while (allImgs.size() > 0);
return result;
}
void calcCtrlPointErrors (PanoramaData& pano)
{
if(pano.getNrOfImages()>0 && pano.getNrOfCtrlPoints()>0)
{
char * p=setlocale(LC_ALL,NULL);
#ifndef ANDROID
char * oldlocale=strdup(p);
#else
char * oldlocale="";
#endif
setlocale(LC_ALL,"C");
UIntSet allImg;
std::ostringstream scriptbuf;
fill_set(allImg,0, unsigned(pano.getNrOfImages()-1));
//create temporary non-empty optimize vector
OptimizeVector optVec;
std::set<std::string> opt;
opt.insert("y");
for(unsigned int i=0;i<pano.getNrOfImages();i++)
{
optVec.push_back(opt);
};
pano.printPanoramaScript(scriptbuf, optVec,
pano.getOptions(), allImg, true);
char * script = 0;
script = strdup(scriptbuf.str().c_str());
AlignInfo ainf;
if (ParseScript( script, &ainf ) == 0)
{
if( CheckParams( &ainf ) == 0 )
{
ainf.fcn = fcnPano;
SetGlobalPtr( &ainf );
pano.updateCtrlPointErrors( GetAlignInfoCtrlPoints(ainf) );
}
}
setlocale(LC_ALL,oldlocale);
free(oldlocale);
};
}
void set(PanoramaData & pano, double x) const
{
pano.updateVariable(m_img,Variable(m_name,x));
}
double get(PanoramaData & pano) const
{
return pano.getImage(m_img).getVar(m_name);
}
void PhotometricOptimizer::optimizePhotometric(PanoramaData & pano, const OptimizeVector & vars,
const std::vector<vigra_ext::PointPairRGB> & correspondences,
AppBase::ProgressReporter & progress,
double & error)
{
OptimizeVector photometricVars;
// keep only the photometric variables
unsigned int optCount=0;
for (OptimizeVector::const_iterator it=vars.begin(); it != vars.end(); ++it)
{
std::set<std::string> cvars;
for (std::set<std::string>::const_iterator itv = (*it).begin();
itv != (*it).end(); ++itv)
{
if ((*itv)[0] == 'E' || (*itv)[0] == 'R' || (*itv)[0] == 'V') {
cvars.insert(*itv);
}
}
photometricVars.push_back(cvars);
optCount+=cvars.size();
}
//if no variables to optimize return
if(optCount==0)
{
return;
};
int nMaxIter = 250;
OptimData data(pano, photometricVars, correspondences, 5/255.0, false, nMaxIter, progress);
int ret;
//double opts[LM_OPTS_SZ];
double info[LM_INFO_SZ];
// parameters
int m=data.m_vars.size();
vigra::ArrayVector<double> p(m, 0.0);
// vector for errors
int n=2*3*correspondences.size()+pano.getNrOfImages();
vigra::ArrayVector<double> x(n, 0.0);
data.ToX(p.begin());
#ifdef DEBUG
printf("Parameters before optimisation: ");
for(int i=0; i<m; ++i)
printf("%.7g ", p[i]);
printf("\n");
#endif
// covariance matrix at solution
vigra::DImage cov(m,m);
// TODO: setup optimisation options with some good defaults.
double optimOpts[5];
optimOpts[0] = 1E-03; // init mu
// stop thresholds
optimOpts[1] = 1e-5; // ||J^T e||_inf
optimOpts[2] = 1e-5; // ||Dp||_2
optimOpts[3] = 1e-1; // ||e||_2
// difference mode
optimOpts[4] = LM_DIFF_DELTA;
ret=dlevmar_dif(&photometricError, &photometricVis, &(p[0]), &(x[0]), m, n, nMaxIter, optimOpts, info, NULL, &(cov(0,0)), &data); // no jacobian
// copy to source images (data.m_imgs)
data.FromX(p.begin());
// calculate error at solution
data.huberSigma = 0;
photometricError(&(p[0]), &(x[0]), m, n, &data);
error = 0;
for (int i=0; i<n; i++) {
error += x[i]*x[i];
}
error = sqrt(error/n);
#ifdef DEBUG
printf("Levenberg-Marquardt returned %d in %g iter, reason %g\nSolution: ", ret, info[5], info[6]);
for(int i=0; i<m; ++i)
printf("%.7g ", p[i]);
printf("\n\nMinimization info:\n");
for(int i=0; i<LM_INFO_SZ; ++i)
printf("%g ", info[i]);
printf("\n");
#endif
// copy settings to panorama
for (unsigned i=0; i<pano.getNrOfImages(); i++) {
pano.setSrcImage(i, data.m_imgs[i]);
}
}
Matrix3 StraightenPanorama::calcStraighteningRotation(const PanoramaData& panorama)
{
// landscape/non rotated portrait detection is not working correctly
// should use the exif rotation tag but thats not stored anywhere currently...
// 1: use y axis (image x axis), for normal image
// 0: use z axis (image y axis), for non rotated portrait images
// (usually rotation is just stored in EXIF tag)
std::vector<int> coord_idx;
for (unsigned int i = 0; i < panorama.getNrOfImages(); i++) {
SrcPanoImage img = panorama.getSrcImage(i);
// BUG: need to read exif data here, since exif orientation is not
// stored in Panorama data model
double fl=0;
double crop=0;
img.readEXIF(fl, crop, false, false);
double roll = img.getExifOrientation();
if (roll == 90 || roll == 270 ) {
coord_idx.push_back(2);
} else {
coord_idx.push_back(1);
}
}
// build covariance matrix of X
Matrix3 cov;
unsigned int nrOfVariableImages=0;
for (unsigned int i = 0; i < panorama.getNrOfImages(); i++)
{
const SrcPanoImage & img=panorama.getImage(i);
if(img.YawisLinked())
{
//only consider images which are not linked with the previous ones
bool consider=true;
for(unsigned int j=0; j<i; j++)
{
if(img.YawisLinkedWith(panorama.getImage(j)))
{
consider=false;
break;
};
};
if(!consider)
continue;
};
double y = const_map_get(panorama.getImageVariables(i), "y").getValue();
double p = const_map_get(panorama.getImageVariables(i), "p").getValue();
double r = const_map_get(panorama.getImageVariables(i), "r").getValue();
Matrix3 mat;
mat.SetRotationPT(DEG_TO_RAD(y), DEG_TO_RAD(p), DEG_TO_RAD(r));
nrOfVariableImages++;
DEBUG_DEBUG("mat = " << mat);
for (int j=0; j<3; j++) {
for (int k=0; k<3; k++) {
cov.m[j][k] += mat.m[j][coord_idx[i]] * mat.m[k][coord_idx[i]];
}
}
}
cov /= nrOfVariableImages;
DEBUG_DEBUG("cov = " << cov);
// calculate eigenvalues and vectors
Matrix3 eigvectors;
double eigval[3];
int eigvalIdx[3];
int maxsweep = 100;
int maxannil = 0;
double eps = 1e-16;
hugin_utils::eig_jacobi(3, cov.m, eigvectors.m, eigval, eigvalIdx, &maxsweep, &maxannil, &eps);
DEBUG_DEBUG("Eigenvectors & eigenvalues:" << std::endl
<< "V = " << eigvectors << std::endl
<< "D = [" << eigval[0] << ", " << eigval[1] << ", " << eigval[2] << " ]"
<< "idx = [" << eigvalIdx[0] << ", " << eigvalIdx[1] << ", " << eigvalIdx[2] << " ]");
// get up vector, eigenvector with smallest eigenvalue
Vector3 up;
up.x = eigvectors.m[eigvalIdx[2]][0];
up.y = eigvectors.m[eigvalIdx[2]][1];
up.z = eigvectors.m[eigvalIdx[2]][2];
// normalize vector
up.Normalize();
DEBUG_DEBUG("Up vector: up = " << up );
double rotAngle = acos(up.Dot(Vector3(0,0,1)));
if (rotAngle > M_PI/2) {
// turn in shorter direction
up *= -1;
rotAngle = acos(up.Dot(Vector3(0,0,1)));
}
DEBUG_DEBUG("rotation Angle: " << rotAngle);
// get rotation axis
Vector3 rotAxis = up.Cross(Vector3(0,0,1));
DEBUG_DEBUG("rotAxis = " << rotAngle);
// calculate rotation matrix
Matrix3 rotMat = GetRotationAroundU(rotAxis, -rotAngle);
DEBUG_DEBUG("rotMat = " << rotMat);
return rotMat;
}
FDiff2D CalculateFOV::calcFOV(const PanoramaData& panorama)
{
if (panorama.getNrOfImages() == 0) {
// no change
return FDiff2D(panorama.getOptions().getHFOV(), panorama.getOptions().getVFOV());
}
vigra::Size2D panoSize(360*2,180*2);
// remap into minature pano.
PanoramaOptions opts;
opts.setHFOV(360);
opts.setProjection(PanoramaOptions::EQUIRECTANGULAR);
opts.setWidth(panoSize.x);
opts.setHeight(panoSize.y);
// remap image
// DGSW - make sure the type is correct
vigra::BImage panoAlpha(panoSize.x, panoSize.y,static_cast< unsigned char >(0));
// vigra::BImage panoAlpha(panoSize.x, panoSize.y,0);
Nona::RemappedPanoImage<vigra::BImage, vigra::BImage> remapped;
UIntSet activeImgs = panorama.getActiveImages();
for (UIntSet::iterator it = activeImgs.begin(); it != activeImgs.end(); ++it) {
// for (unsigned int imgNr=0; imgNr < getNrOfImages(); imgNr++) {
// DGSW FIXME - Unreferenced
// const PanoImage & img = getImage(*it);
remapped.setPanoImage(panorama.getSrcImage(*it), opts, vigra::Rect2D(0,0,panoSize.x,panoSize.y));
//remapped.setPanoImage(*this, *it, vigra::Size2D(img.getWidth(), img.getHeight()), opts);
// calculate alpha channel
remapped.calcAlpha();
// copy into global alpha channel.
vigra::copyImageIf(vigra_ext::applyRect(remapped.boundingBox(),
vigra_ext::srcMaskRange(remapped)),
vigra_ext::applyRect(remapped.boundingBox(),
vigra_ext::srcMask(remapped)),
vigra_ext::applyRect(remapped.boundingBox(),
destImage(panoAlpha)));
// vigra::ImageExportInfo imge2("c:/hugin_calcfov_alpha.png");
// exportImage(vigra::srcImageRange(panoAlpha), imge2);
}
// get field of view
FDiff2D ul,lr;
bool found = false;
ul.x = DBL_MAX;
ul.y = DBL_MAX;
lr.x = -DBL_MAX;
lr.y = -DBL_MAX;
for (int v=0; v< panoSize.y; v++) {
for (int h=0; h < panoSize.x; h++) {
if (panoAlpha(h,v)) {
// pixel is valid
if ( ul.x > h ) {
found=true;
ul.x = h;
}
if ( ul.y > v ) {
found=true;
ul.y = v;
}
if ( lr.x < h) {
found=true;
lr.x = h;
}
if ( lr.y < v) {
found=true;
lr.y = v;
}
}
}
}
if (!found) {
// if nothing found, return current fov
return FDiff2D(panorama.getOptions().getHFOV(), panorama.getOptions().getVFOV());
}
ul=ul/2.0;
lr=lr/2.0;
ul.x = ul.x - 180;
ul.y = ul.y - 90;
lr.x = lr.x - 180;
lr.y = lr.y - 90;
FDiff2D fov (2*std::max(fabs(ul.x), fabs(lr.x)), 2*std::max(fabs(ul.y), fabs(lr.y)));
if(fov.x<40)
{
fov.x+=1;
};
return fov;
}
void CenterHorizontally::centerHorizontically(PanoramaData& panorama)
{
vigra::Size2D panoSize(360,180);
// remap into minature pano.
PanoramaOptions opts;
opts.setHFOV(360);
opts.setProjection(PanoramaOptions::EQUIRECTANGULAR);
opts.setWidth(360);
opts.setHeight(180);
// remap image
vigra::BImage panoAlpha(panoSize);
Nona::RemappedPanoImage<vigra::BImage, vigra::BImage> remapped;
// use selected images.
const UIntSet allActiveImgs(panorama.getActiveImages());
if (allActiveImgs.empty())
{
// do nothing if there are no images
return;
}
//only check unlinked images
UIntSet activeImgs;
for (UIntSet::const_iterator it = allActiveImgs.begin(); it!= allActiveImgs.end(); ++it)
{
const SrcPanoImage & img=panorama.getImage(*it);
bool consider=true;
if(img.YawisLinked())
{
for(UIntSet::const_iterator it2=activeImgs.begin(); it2!=activeImgs.end(); ++it2)
{
if(img.YawisLinkedWith(panorama.getSrcImage(*it2)))
{
consider=false;
break;
};
};
};
if(consider)
activeImgs.insert(*it);
};
for (UIntSet::iterator it = activeImgs.begin(); it != activeImgs.end(); ++it)
{
remapped.setPanoImage(panorama.getSrcImage(*it), opts, vigra::Rect2D(0,0,360,180));
// calculate alpha channel
remapped.calcAlpha();
// copy into global alpha channel.
vigra::copyImageIf(vigra_ext::applyRect(remapped.boundingBox(),
vigra_ext::srcMaskRange(remapped)),
vigra_ext::applyRect(remapped.boundingBox(),
vigra_ext::srcMask(remapped)),
vigra_ext::applyRect(remapped.boundingBox(),
destImage(panoAlpha)));
}
// get field of view
std::vector<int> borders;
bool colOccupied = false;
for (int h=0; h < 360; h++) {
bool curColOccupied = false;
for (int v=0; v< 180; v++) {
if (panoAlpha(h,v)) {
// pixel is valid
curColOccupied = true;
}
}
if ((colOccupied && !curColOccupied) ||
(!colOccupied && curColOccupied))
{
// change in position, save point.
borders.push_back(h-180);
colOccupied = curColOccupied;
}
}
int lastidx = borders.size() -1;
if (lastidx == -1) {
// empty pano
return;
}
if (colOccupied) {
// we have reached the right border, and the pano is still valid
// shift right fragments by 360 deg
// |11 2222| -> | 222211 |
std::vector<int> newBorders;
newBorders.push_back(borders[lastidx]);
for (int i = 0; i < lastidx; i++) {
newBorders.push_back(borders[i]+360);
}
borders = newBorders;
}
const double dYaw=(borders[0] + borders[lastidx])/2;
// apply yaw shift, takes also translation parameters into account
RotatePanorama(panorama, -dYaw, 0, 0).run();
}
