OptimizeVector OptimizePhotometricPanel::getOptimizeVector() { int nrLI = m_exp_list->GetCount(); assert(nrLI >=0); unsigned int nr = (unsigned int) nrLI; unsigned int nImages = m_pano->getNrOfImages(); assert(nr == nImages); OptimizeVector optvars; for (unsigned int i=0; i < nImages; i++) { set<string> imgopt; // lens variables unsigned int lensNr = variable_groups->getLenses().getPartNumber(i); if (m_vig_list->IsChecked(lensNr)) { imgopt.insert("Vb"); imgopt.insert("Vc"); imgopt.insert("Vd"); } if (m_vigc_list->IsChecked(lensNr)) { imgopt.insert("Vx"); imgopt.insert("Vy"); } if (m_resp_list->IsChecked(lensNr)) { imgopt.insert("Ra"); imgopt.insert("Rb"); imgopt.insert("Rc"); imgopt.insert("Rd"); imgopt.insert("Re"); } // image variables if (m_exp_list->IsChecked(i)) { imgopt.insert("Eev"); } if (m_wb_list->IsChecked(i)) { imgopt.insert("Er"); imgopt.insert("Eb"); } optvars.push_back(imgopt); } return optvars; }
void calcCtrlPointErrors (PanoramaData& pano) { if(pano.getNrOfImages()>0 && pano.getNrOfCtrlPoints()>0) { char * p=setlocale(LC_ALL,NULL); #ifndef ANDROID char * oldlocale=strdup(p); #else char * oldlocale=""; #endif setlocale(LC_ALL,"C"); UIntSet allImg; std::ostringstream scriptbuf; fill_set(allImg,0, unsigned(pano.getNrOfImages()-1)); //create temporary non-empty optimize vector OptimizeVector optVec; std::set<std::string> opt; opt.insert("y"); for(unsigned int i=0;i<pano.getNrOfImages();i++) { optVec.push_back(opt); }; pano.printPanoramaScript(scriptbuf, optVec, pano.getOptions(), allImg, true); char * script = 0; script = strdup(scriptbuf.str().c_str()); AlignInfo ainf; if (ParseScript( script, &ainf ) == 0) { if( CheckParams( &ainf ) == 0 ) { ainf.fcn = fcnPano; SetGlobalPtr( &ainf ); pano.updateCtrlPointErrors( GetAlignInfoCtrlPoints(ainf) ); } } setlocale(LC_ALL,oldlocale); free(oldlocale); }; }
void PhotometricOptimizer::optimizePhotometric(PanoramaData & pano, const OptimizeVector & vars, const std::vector<vigra_ext::PointPairRGB> & correspondences, AppBase::ProgressReporter & progress, double & error) { OptimizeVector photometricVars; // keep only the photometric variables unsigned int optCount=0; for (OptimizeVector::const_iterator it=vars.begin(); it != vars.end(); ++it) { std::set<std::string> cvars; for (std::set<std::string>::const_iterator itv = (*it).begin(); itv != (*it).end(); ++itv) { if ((*itv)[0] == 'E' || (*itv)[0] == 'R' || (*itv)[0] == 'V') { cvars.insert(*itv); } } photometricVars.push_back(cvars); optCount+=cvars.size(); } //if no variables to optimize return if(optCount==0) { return; }; int nMaxIter = 250; OptimData data(pano, photometricVars, correspondences, 5/255.0, false, nMaxIter, progress); int ret; //double opts[LM_OPTS_SZ]; double info[LM_INFO_SZ]; // parameters int m=data.m_vars.size(); vigra::ArrayVector<double> p(m, 0.0); // vector for errors int n=2*3*correspondences.size()+pano.getNrOfImages(); vigra::ArrayVector<double> x(n, 0.0); data.ToX(p.begin()); #ifdef DEBUG printf("Parameters before optimisation: "); for(int i=0; i<m; ++i) printf("%.7g ", p[i]); printf("\n"); #endif // covariance matrix at solution vigra::DImage cov(m,m); // TODO: setup optimisation options with some good defaults. double optimOpts[5]; optimOpts[0] = 1E-03; // init mu // stop thresholds optimOpts[1] = 1e-5; // ||J^T e||_inf optimOpts[2] = 1e-5; // ||Dp||_2 optimOpts[3] = 1e-1; // ||e||_2 // difference mode optimOpts[4] = LM_DIFF_DELTA; ret=dlevmar_dif(&photometricError, &photometricVis, &(p[0]), &(x[0]), m, n, nMaxIter, optimOpts, info, NULL, &(cov(0,0)), &data); // no jacobian // copy to source images (data.m_imgs) data.FromX(p.begin()); // calculate error at solution data.huberSigma = 0; photometricError(&(p[0]), &(x[0]), m, n, &data); error = 0; for (int i=0; i<n; i++) { error += x[i]*x[i]; } error = sqrt(error/n); #ifdef DEBUG printf("Levenberg-Marquardt returned %d in %g iter, reason %g\nSolution: ", ret, info[5], info[6]); for(int i=0; i<m; ++i) printf("%.7g ", p[i]); printf("\n\nMinimization info:\n"); for(int i=0; i<LM_INFO_SZ; ++i) printf("%g ", info[i]); printf("\n"); #endif // copy settings to panorama for (unsigned i=0; i<pano.getNrOfImages(); i++) { pano.setSrcImage(i, data.m_imgs[i]); } }
void OptimizePhotometricPanel::runOptimizer(const UIntSet & imgs) { DEBUG_TRACE(""); int mode = m_mode_cb->GetSelection(); // check if vignetting and response are linked, display a warning if they are not // The variables to check: const HuginBase::ImageVariableGroup::ImageVariableEnum vars[] = { HuginBase::ImageVariableGroup::IVE_EMoRParams, HuginBase::ImageVariableGroup::IVE_ResponseType, HuginBase::ImageVariableGroup::IVE_VigCorrMode, HuginBase::ImageVariableGroup::IVE_RadialVigCorrCoeff, HuginBase::ImageVariableGroup::IVE_RadialVigCorrCenterShift }; // keep a list of commands needed to fix it: std::vector<PT::PanoCommand *> commands; HuginBase::ConstImageVariableGroup & lenses = variable_groups->getLenses(); for (size_t i = 0; i < lenses.getNumberOfParts(); i++) { std::set<HuginBase::ImageVariableGroup::ImageVariableEnum> links_needed; links_needed.clear(); for (int v = 0; v < 5; v++) { if (!lenses.getVarLinkedInPart(vars[v], i)) { links_needed.insert(vars[v]); } }; if (!links_needed.empty()) { commands.push_back(new PT::LinkLensVarsCmd(*m_pano, i, links_needed)); } } // if the list of commands is empty, all is good and we don't need a warning. if (!commands.empty()) { int ok = wxMessageBox(_("The same vignetting and response parameters should\nbe applied for all images of a lens.\nCurrently each image can have different parameters.\nLink parameters?"), _("Link parameters"), wxYES_NO | wxICON_INFORMATION); if (ok == wxYES) { // perform all the commands we stocked up earilier. for (std::vector<PT::PanoCommand *>::iterator it = commands.begin(); it != commands.end(); it++) { GlobalCmdHist::getInstance().addCommand(*it); } } else { // free all the commands, the user doesn't want them used. for (std::vector<PT::PanoCommand *>::iterator it = commands.begin(); it != commands.end(); it++) { delete *it; } } } Panorama optPano = m_pano->getSubset(imgs); PanoramaOptions opts = optPano.getOptions(); OptimizeVector optvars; if(mode==OPT_CUSTOM) { optvars = getOptimizeVector(); if (optPano.getNrOfImages() != m_pano->getNrOfImages()) { OptimizeVector o = optvars; optvars.clear(); for (UIntSet::const_iterator it = imgs.begin(); it != imgs.end(); ++it) { optvars.push_back(o[*it]); } } unsigned int countVar=0; for(unsigned int i=0;i<optvars.size();i++) { countVar+=optvars[i].size(); }; if(countVar==0) { wxMessageBox(_("You selected no parameters to optimize.\nTherefore optimization will be canceled."), _("Exposure optimization"), wxOK | wxICON_INFORMATION); return; }; }; std::vector<vigra_ext::PointPairRGB> m_points; // extract points only if not done previously long nPoints = 200; wxConfigBase::Get()->Read(wxT("/OptimizePhotometric/nRandomPointsPerImage"), & nPoints); // get parameters for estimation. nPoints = wxGetNumberFromUser(_("The vignetting and exposure correction is determined by analysing color values in the overlapping areas.\nTo speed up the computation, only a random subset of points is used."), _("Number of points per image"), _("Photometric optimization"), nPoints, 0, 32000, this); if (nPoints < 0) { return; } wxConfigBase::Get()->Write(wxT("/OptimizePhotometric/nRandomPointsPerImage"),nPoints); ProgressReporterDialog progress(5.0, _("Photometric alignment"), _("Loading images")); progress.Show(); nPoints = nPoints * optPano.getNrOfImages(); // get the small images std::vector<vigra::FRGBImage *> srcImgs; for (size_t i=0; i < optPano.getNrOfImages(); i++) { ImageCache::EntryPtr e = ImageCache::getInstance().getSmallImage(optPano.getImage(i).getFilename()); vigra::FRGBImage * img = new FRGBImage; if (!e) { wxMessageBox(_("Error: could not load all images"), _("Error")); return; } if (e->image8 && e->image8->width() > 0) { reduceToNextLevel(*(e->image8), *img); transformImage(vigra::srcImageRange(*img), vigra::destImage(*img), vigra::functor::Arg1()/vigra::functor::Param(255.0)); } else if (e->image16 && e->image16->width() > 0) { reduceToNextLevel(*(e->image16), *img); transformImage(vigra::srcImageRange(*img), vigra::destImage(*img), vigra::functor::Arg1()/vigra::functor::Param(65535.0)); } else { reduceToNextLevel(*(e->imageFloat), *img); } srcImgs.push_back(img); } bool randomPoints = true; extractPoints(optPano, srcImgs, nPoints, randomPoints, progress, m_points); if (m_points.size() == 0) { wxMessageBox(_("Error: no overlapping points found, Photometric optimization aborted"), _("Error")); return; } double error = 0; try { //wxBusyCursor busyc; if (mode != OPT_CUSTOM) { // run automatic optimisation // ensure that we have a valid anchor. PanoramaOptions opts = optPano.getOptions(); if (opts.colorReferenceImage >= optPano.getNrOfImages()) { opts.colorReferenceImage = 0; optPano.setOptions(opts); } smartOptimizePhotometric(optPano, PhotometricOptimizeMode(mode), m_points, progress, error); } else { // optimize selected parameters optimizePhotometric(optPano, optvars, m_points, progress, error); } } catch (std::exception & error) { wxMessageBox(_("Internal error during photometric optimization:\n") + wxString(error.what(), wxConvLocal), _("Internal error")); } progress.Close(); // display information about the estimation process: int ret = wxMessageBox(wxString::Format(_("Photometric optimization results:\nAverage difference (RMSE) between overlapping pixels: %.2f gray values (0..255)\n\nApply results?"), error*255), _("Photometric optimization finished"), wxYES_NO | wxICON_INFORMATION,this); if (ret == wxYES) { DEBUG_DEBUG("Applying vignetting corr"); // TODO: merge into a single update command const VariableMapVector & vars = optPano.getVariables(); GlobalCmdHist::getInstance().addCommand( new PT::UpdateImagesVariablesCmd(*m_pano, imgs, vars) ); //now update panorama exposure value PanoramaOptions opts = m_pano->getOptions(); opts.outputExposureValue = calcMeanExposure(*m_pano); GlobalCmdHist::getInstance().addCommand( new PT::SetPanoOptionsCmd(*m_pano, opts) ); } }
bool PanoDetector::matchMultiRow(PoolExecutor& aExecutor) { //step 1 std::vector<HuginBase::UIntSet> checkedImagePairs(_panoramaInfo->getNrOfImages()); std::vector<stack_img> stack_images; HuginBase::StandardImageVariableGroups* variable_groups = new HuginBase::StandardImageVariableGroups(*_panoramaInfo); for(unsigned int i=0; i<_panoramaInfo->getNrOfImages(); i++) { unsigned int stack_nr=variable_groups->getStacks().getPartNumber(i); //check, if this stack is already in list bool found=false; unsigned int index=0; for(index=0; index<stack_images.size(); index++) { found=(stack_images[index].layer_nr==stack_nr); if(found) { break; }; }; if(!found) { //new stack stack_images.resize(stack_images.size()+1); index=stack_images.size()-1; //add new stack stack_images[index].layer_nr=stack_nr; }; //add new image unsigned int new_image_index=stack_images[index].images.size(); stack_images[index].images.resize(new_image_index+1); stack_images[index].images[new_image_index].img_nr=i; stack_images[index].images[new_image_index].ev=_panoramaInfo->getImage(i).getExposure(); }; delete variable_groups; //get image with median exposure for search with cp generator vector<size_t> images_layer; UIntSet images_layer_set; for(unsigned int i=0; i<stack_images.size(); i++) { std::sort(stack_images[i].images.begin(),stack_images[i].images.end(),sort_img_ev); unsigned int index=0; if(stack_images[i].images[0].ev!=stack_images[i].images[stack_images[i].images.size()-1].ev) { index=stack_images[i].images.size() / 2; }; images_layer.push_back(stack_images[i].images[index].img_nr); images_layer_set.insert(stack_images[i].images[index].img_nr); if(stack_images[i].images.size()>1) { //build match list for stacks for(unsigned int j=0; j<stack_images[i].images.size()-1; j++) { size_t img1=stack_images[i].images[j].img_nr; size_t img2=stack_images[i].images[j+1].img_nr; _matchesData.push_back(MatchData()); MatchData& aM=_matchesData.back(); aM._i1=&(_filesData[img1]); aM._i2=&(_filesData[img2]); checkedImagePairs[img1].insert(img2); checkedImagePairs[img2].insert(img1); }; }; }; //build match data list for image pairs if(images_layer.size()>1) { std::sort(images_layer.begin(), images_layer.end()); for(unsigned int i=0; i<images_layer.size()-1; i++) { size_t img1=images_layer[i]; size_t img2=images_layer[i+1]; _matchesData.push_back(MatchData()); MatchData& aM = _matchesData.back(); aM._i1 = &(_filesData[img1]); aM._i2 = &(_filesData[img2]); checkedImagePairs[img1].insert(img2); checkedImagePairs[img2].insert(img1); }; }; TRACE_INFO(endl<< "--- Find matches ---" << endl); try { BOOST_FOREACH(MatchData& aMD, _matchesData) aExecutor.execute(new MatchDataRunnable(aMD, *this)); aExecutor.wait(); } catch(Synchronization_Exception& e) { TRACE_ERROR(e.what() << endl); return false; } // Add detected matches to _panoramaInfo BOOST_FOREACH(MatchData& aM, _matchesData) BOOST_FOREACH(lfeat::PointMatchPtr& aPM, aM._matches) _panoramaInfo->addCtrlPoint(ControlPoint(aM._i1->_number, aPM->_img1_x, aPM->_img1_y, aM._i2->_number, aPM->_img2_x, aPM->_img2_y)); // step 2: connect all image groups _matchesData.clear(); Panorama mediumPano=_panoramaInfo->getSubset(images_layer_set); CPGraph graph; createCPGraph(mediumPano, graph); CPComponents comps; unsigned int n = findCPComponents(graph, comps); if(n>1) { vector<unsigned int> ImagesGroups; for(unsigned int i=0; i<n; i++) { ImagesGroups.push_back(images_layer[*(comps[i].begin())]); if(comps[i].size()>1) { ImagesGroups.push_back(images_layer[*(comps[i].rbegin())]); } }; for(unsigned int i=0; i<ImagesGroups.size()-1; i++) { for(unsigned int j=i+1; j<ImagesGroups.size(); j++) { size_t img1=ImagesGroups[i]; size_t img2=ImagesGroups[j]; //skip already checked image pairs if(set_contains(checkedImagePairs[img1],img2)) { continue; }; _matchesData.push_back(MatchData()); MatchData& aM = _matchesData.back(); aM._i1 = &(_filesData[img1]); aM._i2 = &(_filesData[img2]); checkedImagePairs[img1].insert(img2); checkedImagePairs[img2].insert(img1); }; }; TRACE_INFO(endl<< "--- Find matches in images groups ---" << endl); try { BOOST_FOREACH(MatchData& aMD, _matchesData) aExecutor.execute(new MatchDataRunnable(aMD, *this)); aExecutor.wait(); } catch(Synchronization_Exception& e) { TRACE_ERROR(e.what() << endl); return false; } BOOST_FOREACH(MatchData& aM, _matchesData) BOOST_FOREACH(lfeat::PointMatchPtr& aPM, aM._matches) _panoramaInfo->addCtrlPoint(ControlPoint(aM._i1->_number, aPM->_img1_x, aPM->_img1_y, aM._i2->_number, aPM->_img2_x, aPM->_img2_y)); }; // step 3: now connect all overlapping images _matchesData.clear(); PT::Panorama optPano=_panoramaInfo->getSubset(images_layer_set); createCPGraph(optPano, graph); if(findCPComponents(graph, comps)==1) { if(images_layer.size()>2) { //reset translation parameters VariableMapVector varMapVec=optPano.getVariables(); for(size_t i=0; i<varMapVec.size(); i++) { map_get(varMapVec[i], "TrX").setValue(0); map_get(varMapVec[i], "TrY").setValue(0); map_get(varMapVec[i], "TrZ").setValue(0); }; optPano.updateVariables(varMapVec); //next steps happens only when all images are connected; //now optimize panorama PanoramaOptions opts = optPano.getOptions(); opts.setProjection(PanoramaOptions::EQUIRECTANGULAR); opts.optimizeReferenceImage=0; // calculate proper scaling, 1:1 resolution. // Otherwise optimizer distances are meaningless. opts.setWidth(30000, false); opts.setHeight(15000); optPano.setOptions(opts); int w = optPano.calcOptimalWidth(); opts.setWidth(w); opts.setHeight(w/2); optPano.setOptions(opts); //generate optimize vector, optimize only yaw and pitch OptimizeVector optvars; const SrcPanoImage& anchorImage = optPano.getImage(opts.optimizeReferenceImage); for (unsigned i=0; i < optPano.getNrOfImages(); i++) { std::set<std::string> imgopt; if(i==opts.optimizeReferenceImage) { //optimize only anchors pitch, not yaw imgopt.insert("p"); } else { imgopt.insert("p"); imgopt.insert("y"); }; optvars.push_back(imgopt); } optPano.setOptimizeVector(optvars); // remove vertical and horizontal control points CPVector cps = optPano.getCtrlPoints(); CPVector newCP; for (CPVector::const_iterator it = cps.begin(); it != cps.end(); it++) { if (it->mode == ControlPoint::X_Y) { newCP.push_back(*it); } } optPano.setCtrlPoints(newCP); if (getVerbose() < 2) { PT_setProgressFcn(ptProgress); PT_setInfoDlgFcn(ptinfoDlg); }; //in a first step do a pairwise optimisation HuginBase::AutoOptimise::autoOptimise(optPano, false); //now the final optimisation HuginBase::PTools::optimize(optPano); if (getVerbose() < 2) { PT_setProgressFcn(NULL); PT_setInfoDlgFcn(NULL); }; //now match overlapping images if(!matchPrealigned(aExecutor, &optPano, checkedImagePairs, images_layer, false)) { return false; }; }; } else { if(!match(aExecutor, checkedImagePairs)) { return false; }; }; return true; };
CPVector AutoPanoSiftMultiRow::automatch(CPDetectorSetting &setting, Panorama & pano, const UIntSet & imgs, int nFeatures, int & ret_value, wxWindow *parent) { CPVector cps; if (imgs.size() < 2) { return cps; }; std::vector<wxString> keyFiles(pano.getNrOfImages()); //generate cp for every consecutive image pair unsigned int counter=0; for(UIntSet::const_iterator it = imgs.begin(); it != imgs.end(); ) { if(counter==imgs.size()-1) break; counter++; UIntSet ImagePair; ImagePair.clear(); ImagePair.insert(*it); it++; ImagePair.insert(*it); AutoPanoSift matcher; CPVector new_cps; new_cps.clear(); if(setting.IsTwoStepDetector()) new_cps=matcher.automatch(setting, pano, ImagePair, nFeatures, keyFiles, ret_value, parent); else new_cps=matcher.automatch(setting, pano, ImagePair, nFeatures, ret_value, parent); if(new_cps.size()>0) AddControlPointsWithCheck(cps,new_cps); if(ret_value!=0) { Cleanup(setting, pano, imgs, keyFiles, parent); return cps; }; }; // now connect all image groups // generate temporary panorama to add all found cps UIntSet allImgs; fill_set(allImgs, 0, pano.getNrOfImages()-1); Panorama optPano=pano.getSubset(allImgs); for (CPVector::const_iterator it=cps.begin();it!=cps.end();++it) optPano.addCtrlPoint(*it); CPGraph graph; createCPGraph(optPano, graph); CPComponents comps; int n = findCPComponents(graph, comps); if(n>1) { UIntSet ImagesGroups; for(unsigned int i=0;i<n;i++) { ImagesGroups.insert(*(comps[i].begin())); if(comps[i].size()>1) ImagesGroups.insert(*(comps[i].rbegin())); }; AutoPanoSift matcher; CPVector new_cps; if(setting.IsTwoStepDetector()) new_cps=matcher.automatch(setting, optPano, ImagesGroups, nFeatures, keyFiles, ret_value, parent); else new_cps=matcher.automatch(setting, optPano, ImagesGroups, nFeatures, ret_value, parent); if(new_cps.size()>0) AddControlPointsWithCheck(cps,new_cps,&optPano); if(ret_value!=0) { Cleanup(setting, pano, imgs, keyFiles, parent); return cps; }; createCPGraph(optPano,graph); n=findCPComponents(graph, comps); }; if(n==1 && setting.GetOption()) { //next steps happens only when all images are connected; //now optimize panorama PanoramaOptions opts = pano.getOptions(); opts.setProjection(PanoramaOptions::EQUIRECTANGULAR); // calculate proper scaling, 1:1 resolution. // Otherwise optimizer distances are meaningless. opts.setWidth(30000, false); opts.setHeight(15000); optPano.setOptions(opts); int w = optPano.calcOptimalWidth(); opts.setWidth(w); opts.setHeight(w/2); optPano.setOptions(opts); //generate optimize vector, optimize only yaw and pitch OptimizeVector optvars; const SrcPanoImage & anchorImage = optPano.getImage(opts.optimizeReferenceImage); for (unsigned i=0; i < optPano.getNrOfImages(); i++) { std::set<std::string> imgopt; if(i==opts.optimizeReferenceImage) { //optimize only anchors pitch, not yaw imgopt.insert("p"); } else { // do not optimize anchor image's stack for position. if(!optPano.getImage(i).YawisLinkedWith(anchorImage)) { imgopt.insert("p"); imgopt.insert("y"); }; }; optvars.push_back(imgopt); } optPano.setOptimizeVector(optvars); // remove vertical and horizontal control points CPVector backupOldCPS = optPano.getCtrlPoints(); CPVector backupNewCPS; for (CPVector::const_iterator it = backupOldCPS.begin(); it != backupOldCPS.end(); it++) { if (it->mode == ControlPoint::X_Y) { backupNewCPS.push_back(*it); } } optPano.setCtrlPoints(backupNewCPS); // do a first pairwise optimisation step HuginBase::AutoOptimise::autoOptimise(optPano,false); HuginBase::PTools::optimize(optPano); optPano.setCtrlPoints(backupOldCPS); //and find cp on overlapping images //work only on image pairs, which are not yet connected AutoPanoSiftPreAlign matcher; CPDetectorSetting newSetting; newSetting.SetProg(setting.GetProg()); newSetting.SetArgs(setting.GetArgs()); if(setting.IsTwoStepDetector()) { newSetting.SetProgMatcher(setting.GetProgMatcher()); newSetting.SetArgsMatcher(setting.GetArgsMatcher()); }; newSetting.SetOption(true); CPVector new_cps; if(setting.IsTwoStepDetector()) new_cps=matcher.automatch(newSetting, optPano, imgs, nFeatures, keyFiles, ret_value, parent); else new_cps=matcher.automatch(newSetting, optPano, imgs, nFeatures, ret_value, parent); if(new_cps.size()>0) AddControlPointsWithCheck(cps,new_cps); }; Cleanup(setting, pano, imgs, keyFiles, parent); return cps; };