JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_setWarping( JNIEnv * env, jobject obj, jboolean flag) { // TODO: Review this logic if(gWarpImage != (bool) flag) //switching from viewfinder to capture or vice-versa { // Clear gBuffer[0] gWarper1.SetupGraphics(&gBuffer[0]); gWarper1.Clear(0.0, 0.0, 0.0, 1.0); // Clear gBuffer[1] gWarper1.SetupGraphics(&gBuffer[1]); gWarper1.Clear(0.0, 0.0, 0.0, 1.0); // Clear the screen to black. gPreview.Clear(0.0, 0.0, 0.0, 1.0); gLastTx = 0.0f; gPanOffset = 0.0f; gPanViewfinder = true; db_Identity3x3(gThisH1t); db_Identity3x3(gLastH1t); } gWarpImage = (bool)flag; }
JNIEXPORT void JNICALL Java_org_cyanogenmod_focal_pano_MosaicRenderer_setWarping( JNIEnv * env, jobject obj, jboolean flag) { // TODO: Review this logic if(gWarpImage != (bool) flag) //switching from viewfinder to capture or vice-versa { // Clear gBuffer[0] gWarper1.SetupGraphics(&gBuffer[0]); gWarper1.Clear(0.0, 0.0, 0.0, 1.0); // Clear gBuffer[1] gWarper1.SetupGraphics(&gBuffer[1]); gWarper1.Clear(0.0, 0.0, 0.0, 1.0); // Clear the screen to black. gPreview.Clear(0.0, 0.0, 0.0, 1.0); gLastTx = 0.0f; gPanOffset = 0.0f; gPanViewfinder = true; db_Identity3x3(gThisH1t); db_Identity3x3(gLastH1t); // Make sure g_dAffinetransGL and g_dAffinetransPanGL are updated. // Otherwise, the first frame after setting the flag to true will be // incorrectly drawn. if ((bool) flag) { UpdateWarpTransformation(g_dIdent3x3); } } gWarpImage = (bool)flag; }
Align::Align() { width = height = 0; frame_number = 0; num_frames_captured = 0; reference_frame_index = 0; db_Identity3x3(Hcurr); db_Identity3x3(Hprev); }
void db_FrameToReferenceRegistration::Set_H_dref_to_ins(double H[9]) { double H_ins_to_ref[9]; db_Identity3x3(H_ins_to_ref); // Ensure it has proper values db_InvertAffineTransform(H_ins_to_ref,m_H_ref_to_ins); // Invert to get ins to ref db_Multiply3x3_3x3(m_H_dref_to_ref,H,H_ins_to_ref); // Update dref to ref using the input H from dref to ins }
db_FrameToReferenceRegistration::db_FrameToReferenceRegistration() : m_initialized(false),m_nr_matches(0),m_over_allocation(256),m_nr_bins(20),m_max_cost_pix(30), m_quarter_resolution(false) { m_reference_image = NULL; m_aligned_ins_image = NULL; m_quarter_res_image = NULL; m_horz_smooth_subsample_image = NULL; m_x_corners_ref = NULL; m_y_corners_ref = NULL; m_x_corners_ins = NULL; m_y_corners_ins = NULL; m_match_index_ref = NULL; m_match_index_ins = NULL; m_inlier_indices = NULL; m_num_inlier_indices = 0; m_temp_double = NULL; m_temp_int = NULL; m_corners_ref = NULL; m_corners_ins = NULL; m_sq_cost = NULL; m_cost_histogram = NULL; profile_string = NULL; db_Identity3x3(m_K); db_Identity3x3(m_H_ref_to_ins); db_Identity3x3(m_H_dref_to_ref); m_sq_cost_computed = false; m_reference_set = false; m_reference_update_period = 0; m_nr_frames_processed = 0; return; }
// Save the reference image, detect features and update the dref-to-ref transformation int db_FrameToReferenceRegistration::UpdateReference(const unsigned char * const * im, bool subsample, bool detect_corners) { double temp[9]; db_Multiply3x3_3x3(temp,m_H_dref_to_ref,m_H_ref_to_ins); db_Copy9(m_H_dref_to_ref,temp); const unsigned char * const * imptr = im; if (m_quarter_resolution && subsample) { GenerateQuarterResImage(im); imptr = m_quarter_res_image; } // save the reference image, detect features and quit db_CopyImage_u(m_reference_image,imptr,m_im_width,m_im_height,m_over_allocation); if(detect_corners) { #if MB m_cd.DetectCorners(imptr, m_x_corners_ref,m_y_corners_ref,&m_nr_corners_ref); int nr = 0; for(int k=0; k<m_nr_corners_ref; k++) { if(m_x_corners_ref[k]>m_im_width/3) { m_x_corners_ref[nr] = m_x_corners_ref[k]; m_y_corners_ref[nr] = m_y_corners_ref[k]; nr++; } } m_nr_corners_ref = nr; #else m_cd.DetectCorners(imptr, m_x_corners_ref,m_y_corners_ref,&m_nr_corners_ref); #endif } else { m_nr_corners_ref = m_nr_corners_ins; for(int k=0; k<m_nr_corners_ins; k++) { m_x_corners_ref[k] = m_x_corners_ins[k]; m_y_corners_ref[k] = m_y_corners_ins[k]; } } db_Identity3x3(m_H_ref_to_ins); m_max_inlier_count = 0; // Reset to 0 as no inliers seen until now m_sq_cost_computed = false; m_reference_set = true; m_current_is_reference = true; return 1; }
int Align::initialize(int width, int height, bool _quarter_res, float _thresh_still) { int nr_corners = DEFAULT_NR_CORNERS; double max_disparity = DEFAULT_MAX_DISPARITY; int motion_model_type = DEFAULT_MOTION_MODEL; int nrsamples = DB_DEFAULT_NR_SAMPLES; double scale = DB_POINT_STANDARDDEV; int chunk_size = DB_DEFAULT_CHUNK_SIZE; int nrhorz = width/48; // Empirically determined number of horizontal int nrvert = height/60; // and vertical buckets for harris corner detection. bool linear_polish = false; unsigned int reference_update_period = DEFAULT_REFERENCE_UPDATE_PERIOD; const bool DEFAULT_USE_SMALLER_MATCHING_WINDOW = false; bool use_smaller_matching_window = DEFAULT_USE_SMALLER_MATCHING_WINDOW; quarter_res = _quarter_res; thresh_still = _thresh_still; frame_number = 0; num_frames_captured = 0; reference_frame_index = 0; db_Identity3x3(Hcurr); db_Identity3x3(Hprev); if (!reg.Initialized()) { reg.Init(width, height, motion_model_type, 20, linear_polish, quarter_res, scale, reference_update_period, false, 0, nrsamples, chunk_size, nr_corners, max_disparity, use_smaller_matching_window, nrhorz, nrvert); } this->width = width; this->height = height; imageGray = ImageUtils::allocateImage(width, height, 1); if (reg.Initialized()) return ALIGN_RET_OK; else return ALIGN_RET_ERROR; }
void AllocateTextureMemory(int widthHR, int heightHR, int widthLR, int heightLR) { gPreviewImageWidth[HR] = widthHR; gPreviewImageHeight[HR] = heightHR; gPreviewImageWidth[LR] = widthLR; gPreviewImageHeight[LR] = heightLR; sem_wait(&gPreviewImage_semaphore); gPreviewImage[LR] = ImageUtils::allocateImage(gPreviewImageWidth[LR], gPreviewImageHeight[LR], 4); gPreviewImage[HR] = ImageUtils::allocateImage(gPreviewImageWidth[HR], gPreviewImageHeight[HR], 4); sem_post(&gPreviewImage_semaphore); gPreviewFBOWidth = PREVIEW_FBO_WIDTH_SCALE * gPreviewImageWidth[HR]; gPreviewFBOHeight = PREVIEW_FBO_HEIGHT_SCALE * gPreviewImageHeight[HR]; // The origin is such that the current frame will sit with its center // at the center of the previewFBO gCenterOffsetX = (gPreviewFBOWidth / 2 - gPreviewImageWidth[HR] / 2); gCenterOffsetY = (gPreviewFBOHeight / 2 - gPreviewImageHeight[HR] / 2); gPanOffset = 0.0f; db_Identity3x3(gThisH1t); db_Identity3x3(gLastH1t); gPanViewfinder = true; int w = gPreviewImageWidth[HR]; int h = gPreviewImageHeight[HR]; int wm = gPreviewFBOWidth; int hm = gPreviewFBOHeight; // K is the transformation to map the canonical [-1,1] vertex coordinate // system to the [0,w] image coordinate system before applying the given // affine transformation trs. gKm[0] = wm / 2.0 - 0.5; gKm[1] = 0.0; gKm[2] = wm / 2.0 - 0.5; gKm[3] = 0.0; gKm[4] = hm / 2.0 - 0.5; gKm[5] = hm / 2.0 - 0.5; gKm[6] = 0.0; gKm[7] = 0.0; gKm[8] = 1.0; gK[0] = w / 2.0 - 0.5; gK[1] = 0.0; gK[2] = w / 2.0 - 0.5; gK[3] = 0.0; gK[4] = h / 2.0 - 0.5; gK[5] = h / 2.0 - 0.5; gK[6] = 0.0; gK[7] = 0.0; gK[8] = 1.0; db_Identity3x3(gKinv); db_InvertCalibrationMatrix(gKinv, gK); db_Identity3x3(gKminv); db_InvertCalibrationMatrix(gKminv, gKm); ////////////////////////////////////////// ////// Compute g_Translation now... ////// ////////////////////////////////////////// double T[9], Tp[9], Ttemp[9]; db_Identity3x3(T); T[2] = gCenterOffsetX; T[5] = gCenterOffsetY; // Tp = inv(K) * T * K db_Identity3x3(Ttemp); db_Multiply3x3_3x3(Ttemp, T, gK); db_Multiply3x3_3x3(Tp, gKinv, Ttemp); ConvertAffine3x3toGL4x4(g_dTranslationToFBOCenter, Tp); UpdateWarpTransformation(g_dIdent3x3); }
// This function computes fills the 4x4 matrices g_dAffinetrans, // and g_dAffinetransPan using the specified 3x3 affine // transformation between the first captured frame and the current frame. // The computed g_dAffinetrans is such that it warps the preview mosaic in // the last frame's coordinate system into the coordinate system of the // current frame. Thus, applying this transformation will create the current // frame mosaic but with the current frame missing. This frame will then be // pasted in by gWarper2 after translating it by g_dTranslationToFBOCenter. // The computed g_dAffinetransPan is such that it offsets the computed preview // mosaic horizontally to make the viewfinder pan within the UI layout. void UpdateWarpTransformation(float *trs) { double H[9], Hp[9], Htemp1[9], Htemp2[9], T[9]; for(int i = 0; i < 9; i++) { gThisH1t[i] = trs[i]; } // Alignment is done based on low-res data. // To render the preview mosaic, the translation of the high-res mosaic is estimated to // H2L_FACTOR x low-res-based tranlation. gThisH1t[2] *= H2L_FACTOR; gThisH1t[5] *= H2L_FACTOR; db_Identity3x3(T); T[2] = -gCenterOffsetX; T[5] = -gCenterOffsetY; // H = ( inv(gThisH1t) * gLastH1t ) * T db_Identity3x3(Htemp1); db_Identity3x3(Htemp2); db_Identity3x3(H); db_InvertAffineTransform(Htemp1, gThisH1t); db_Multiply3x3_3x3(Htemp2, Htemp1, gLastH1t); db_Multiply3x3_3x3(H, Htemp2, T); for(int i = 0; i < 9; i++) { gLastH1t[i] = gThisH1t[i]; } // Move the origin such that the frame is centered in the previewFBO // i.e. H = inv(T) * H H[2] += gCenterOffsetX; H[5] += gCenterOffsetY; // Hp = inv(Km) * H * Km // Km moves the coordinate system from openGL to image pixels so // that the alignment transform H can be applied to them. // inv(Km) moves the coordinate system back to openGL normalized // coordinates so that the shader can correctly render it. db_Identity3x3(Htemp1); db_Multiply3x3_3x3(Htemp1, H, gKm); db_Multiply3x3_3x3(Hp, gKminv, Htemp1); ConvertAffine3x3toGL4x4(g_dAffinetrans, Hp); //////////////////////////////////////////////// ////// Compute g_dAffinetransPan now... ////// //////////////////////////////////////////////// gThisTx = trs[2]; if(gPanViewfinder) { gPanOffset += (gThisTx - gLastTx) * VIEWFINDER_PAN_FACTOR_HORZ; } gLastTx = gThisTx; gPanViewfinder = continuePanningFBO(gPanOffset); db_Identity3x3(H); H[2] = gPanOffset; // Hp = inv(Km) * H * Km db_Identity3x3(Htemp1); db_Multiply3x3_3x3(Htemp1, H, gKm); db_Multiply3x3_3x3(Hp, gKminv, Htemp1); //if (gIsLandscapeOrientation) { ConvertAffine3x3toGL4x4(g_dAffinetransPan, Hp); //} else { // // rotate Hp by 90 degress. // db_Multiply3x3_3x3(Htemp1, gRotation90, Hp); // ConvertAffine3x3toGL4x4(g_dAffinetransPan, Htemp1); // } }
int Align::addFrame(ImageType imageGray_) { int ret_code = ALIGN_RET_OK; // Obtain a vector of pointers to rows in image and pass in to dbreg ImageType *m_rows = ImageUtils::imageTypeToRowPointers(imageGray_, width, height); if (frame_number == 0) { reg.AddFrame(m_rows, Hcurr, true); // Force this to be a reference frame int num_corner_ref = reg.GetNrRefCorners(); if (num_corner_ref < MIN_NR_REF_CORNERS) { return ALIGN_RET_LOW_TEXTURE; } } else { reg.AddFrame(m_rows, Hcurr, false); } // Average translation per frame = // [Translation from Frame0 to Frame(n-1)] / [(n-1)] average_tx_per_frame = (num_frames_captured < 2) ? 0.0 : Hprev[2] / (num_frames_captured - 1); // Increment the captured frame counter if we already have a reference frame num_frames_captured++; if (frame_number != 0) { int num_inliers = reg.GetNrInliers(); if(num_inliers < MIN_NR_INLIERS) { ret_code = ALIGN_RET_FEW_INLIERS; Hcurr[0] = 1.0; Hcurr[1] = 0.0; // Set this as the average per frame translation taking into acccount // the separation of the current frame from the reference frame... Hcurr[2] = -average_tx_per_frame * (num_frames_captured - reference_frame_index); Hcurr[3] = 0.0; Hcurr[4] = 1.0; Hcurr[5] = 0.0; Hcurr[6] = 0.0; Hcurr[7] = 0.0; Hcurr[8] = 1.0; } if(fabs(Hcurr[2])<thresh_still && fabs(Hcurr[5])<thresh_still) // Still camera { return ALIGN_RET_ERROR; } // compute the homography: double Hinv33[3][3]; double Hprev33[3][3]; double Hcurr33[3][3]; // Invert and multiple with previous transformation Matrix33::convert9to33(Hcurr33, Hcurr); Matrix33::convert9to33(Hprev33, Hprev); normProjMat33d(Hcurr33); inv33d(Hcurr33, Hinv33); mult33d(Hcurr33, Hprev33, Hinv33); normProjMat33d(Hcurr33); Matrix9::convert33to9(Hprev, Hcurr33); // Since we have already factored the current transformation // into Hprev, we can reset the Hcurr to identity db_Identity3x3(Hcurr); // Update the reference frame to be the current frame reg.UpdateReference(m_rows,quarter_res,false); // Update the reference frame index reference_frame_index = num_frames_captured; } frame_number++; return ret_code; }
void db_StitchSimilarity3DRaw(double *scale,double R[9],double t[3], double **Xp,double **X,int nr_points,int orientation_preserving, int allow_scaling,int allow_rotation,int allow_translation) { int i; double c[3],cp[3],r[3],rp[3],M[9],s,sp,sc; double Rr[9],score_p,score_r; double *temp,*temp_p; if(allow_translation) { db_PointCentroid3D(c,X,nr_points); db_PointCentroid3D(cp,Xp,nr_points); } else { db_Zero3(c); db_Zero3(cp); } db_Zero9(M); s=sp=0; for(i=0;i<nr_points;i++) { temp= *X++; temp_p= *Xp++; r[0]=(*temp++)-c[0]; r[1]=(*temp++)-c[1]; r[2]=(*temp++)-c[2]; rp[0]=(*temp_p++)-cp[0]; rp[1]=(*temp_p++)-cp[1]; rp[2]=(*temp_p++)-cp[2]; M[0]+=r[0]*rp[0]; M[1]+=r[0]*rp[1]; M[2]+=r[0]*rp[2]; M[3]+=r[1]*rp[0]; M[4]+=r[1]*rp[1]; M[5]+=r[1]*rp[2]; M[6]+=r[2]*rp[0]; M[7]+=r[2]*rp[1]; M[8]+=r[2]*rp[2]; s+=db_sqr(r[0])+db_sqr(r[1])+db_sqr(r[2]); sp+=db_sqr(rp[0])+db_sqr(rp[1])+db_sqr(rp[2]); } /*Compute scale*/ if(allow_scaling) sc=sqrt(db_SafeDivision(sp,s)); else sc=1.0; *scale=sc; /*Compute rotation*/ if(allow_rotation) { if(orientation_preserving) { db_RotationFromMOuterProductSum(R,0,M); } else { /*Try preserving*/ db_RotationFromMOuterProductSum(R,&score_p,M); /*Try reversing*/ M[6]= -M[6]; M[7]= -M[7]; M[8]= -M[8]; db_RotationFromMOuterProductSum(Rr,&score_r,M); if(score_r>score_p) { /*Reverse is better*/ R[0]=Rr[0]; R[1]=Rr[1]; R[2]= -Rr[2]; R[3]=Rr[3]; R[4]=Rr[4]; R[5]= -Rr[5]; R[6]=Rr[6]; R[7]=Rr[7]; R[8]= -Rr[8]; } } } else db_Identity3x3(R); /*Compute translation*/ if(allow_translation) { t[0]=cp[0]-sc*(R[0]*c[0]+R[1]*c[1]+R[2]*c[2]); t[1]=cp[1]-sc*(R[3]*c[0]+R[4]*c[1]+R[5]*c[2]); t[2]=cp[2]-sc*(R[6]*c[0]+R[7]*c[1]+R[8]*c[2]); } else db_Zero3(t); }
int db_FrameToReferenceRegistration::AddFrame(const unsigned char * const * im, double H[9],bool force_reference,bool prewarp) { m_current_is_reference = false; if(!m_reference_set || force_reference) { db_Identity3x3(m_H_ref_to_ins); db_Copy9(H,m_H_ref_to_ins); UpdateReference(im,true,true); return 0; } const unsigned char * const * imptr = im; if (m_quarter_resolution) { if (m_quarter_res_image) { GenerateQuarterResImage(im); } imptr = (const unsigned char * const* )m_quarter_res_image; } double H_last[9]; db_Copy9(H_last,m_H_ref_to_ins); db_Identity3x3(m_H_ref_to_ins); m_sq_cost_computed = false; // detect corners on inspection image and match to reference image features:s // @jke - Adding code to time the functions. TODO: Remove after test #if PROFILE double iTimer1, iTimer2; char str[255]; strcpy(profile_string,"\n"); sprintf(str,"[%dx%d] %p\n",m_im_width,m_im_height,im); strcat(profile_string, str); #endif // @jke - Adding code to time the functions. TODO: Remove after test #if PROFILE iTimer1 = now_ms(); #endif m_cd.DetectCorners(imptr, m_x_corners_ins,m_y_corners_ins,&m_nr_corners_ins); // @jke - Adding code to time the functions. TODO: Remove after test # if PROFILE iTimer2 = now_ms(); double elapsedTimeCorner = iTimer2 - iTimer1; sprintf(str,"Corner Detection [%d corners] = %g ms\n",m_nr_corners_ins, elapsedTimeCorner); strcat(profile_string, str); #endif // @jke - Adding code to time the functions. TODO: Remove after test #if PROFILE iTimer1 = now_ms(); #endif if(prewarp) m_cm.Match(m_reference_image,imptr,m_x_corners_ref,m_y_corners_ref,m_nr_corners_ref, m_x_corners_ins,m_y_corners_ins,m_nr_corners_ins, m_match_index_ref,m_match_index_ins,&m_nr_matches,H,0); else m_cm.Match(m_reference_image,imptr,m_x_corners_ref,m_y_corners_ref,m_nr_corners_ref, m_x_corners_ins,m_y_corners_ins,m_nr_corners_ins, m_match_index_ref,m_match_index_ins,&m_nr_matches); // @jke - Adding code to time the functions. TODO: Remove after test # if PROFILE iTimer2 = now_ms(); double elapsedTimeMatch = iTimer2 - iTimer1; sprintf(str,"Matching [%d] = %g ms\n",m_nr_matches,elapsedTimeMatch); strcat(profile_string, str); #endif // copy out matching features: for ( int i = 0; i < m_nr_matches; ++i ) { int offset = 3*i; m_corners_ref[offset ] = m_x_corners_ref[m_match_index_ref[i]]; m_corners_ref[offset+1] = m_y_corners_ref[m_match_index_ref[i]]; m_corners_ref[offset+2] = 1.0; m_corners_ins[offset ] = m_x_corners_ins[m_match_index_ins[i]]; m_corners_ins[offset+1] = m_y_corners_ins[m_match_index_ins[i]]; m_corners_ins[offset+2] = 1.0; } // @jke - Adding code to time the functions. TODO: Remove after test #if PROFILE iTimer1 = now_ms(); #endif // perform the alignment: db_RobImageHomography(m_H_ref_to_ins, m_corners_ref, m_corners_ins, m_nr_matches, m_K, m_K, m_temp_double, m_temp_int, m_homography_type,NULL,m_max_iterations,m_max_nr_matches,m_scale, m_nr_samples, m_chunk_size); // @jke - Adding code to time the functions. TODO: Remove after test # if PROFILE iTimer2 = now_ms(); double elapsedTimeHomography = iTimer2 - iTimer1; sprintf(str,"Homography = %g ms\n",elapsedTimeHomography); strcat(profile_string, str); #endif SetOutlierThreshold(); // Compute the inliers for the db compute m_H_ref_to_ins ComputeInliers(m_H_ref_to_ins); // Update the max inlier count m_max_inlier_count = (m_max_inlier_count > m_num_inlier_indices)?m_max_inlier_count:m_num_inlier_indices; // Fit a least-squares model to just the inliers and put it in m_H_ref_to_ins if(m_linear_polish) Polish(m_inlier_indices, m_num_inlier_indices); if (m_quarter_resolution) { m_H_ref_to_ins[2] *= 2.0; m_H_ref_to_ins[5] *= 2.0; } #if PROFILE sprintf(str,"#Inliers = %d \n",m_num_inlier_indices); strcat(profile_string, str); #endif /* ///// CHECK IF CURRENT TRANSFORMATION GOOD OR BAD //// ///// IF BAD, then update reference to the last correctly aligned inspection frame; if(m_num_inlier_indices<5)//0.9*m_nr_matches || m_nr_matches < 20) { db_Copy9(m_H_ref_to_ins,H_last); UpdateReference(imptr,false); // UpdateReference(m_aligned_ins_image,false); } else { ///// IF GOOD, then update the last correctly aligned inspection frame to be this; //db_CopyImage_u(m_aligned_ins_image,imptr,m_im_width,m_im_height,m_over_allocation); */ if(m_do_motion_smoothing) SmoothMotion(); // Disable debug printing // db_PrintDoubleMatrix(m_H_ref_to_ins,3,3); db_Copy9(H, m_H_ref_to_ins); m_nr_frames_processed++; { if ( (m_nr_frames_processed % m_reference_update_period) == 0 ) { //UpdateReference(imptr,false, false); #if MB UpdateReference(imptr,false, true); #else UpdateReference(imptr,false, false); #endif } } return 1; }
void db_FrameToReferenceRegistration::ResetDisplayReference() { db_Identity3x3(m_H_dref_to_ref); }