static void PrintTrack(const ImageKeyVector &track)
{
    int num_views = (int) track.size();

    printf("[");
    for (int i = 0; i < num_views; i++) {
	printf(" (%d %d)", track[i].first, track[i].second);
	if (i < num_views - 1)
	    printf(",");
    }
    printf(" ]");
}
/* Add new points to the bundle adjustment */
int BundlerApp::BundleAdjustAddNewPoints(int camera_idx, 
                                         int num_points, int num_cameras,
                                         int *added_order,
                                         camera_params_t *cameras,
                                         v3_t *points, v3_t *colors,
                                         double reference_baseline,
                                         std::vector<ImageKeyVector> &pt_views)
{
    int pt_count = num_points;

    int image_idx = added_order[camera_idx];

    /* Recompute the locations of the new points given the initial
     * pose estimate */
    for (int i = 0; i < num_cameras; i++) {
	int other = added_order[i];

	if (other == image_idx)
	    continue;

	int first = MIN(image_idx, other);
	int second = MAX(image_idx, other);

	MatchIndex idx = GetMatchIndex(first, second);

        SetMatchesFromTracks(first, second);

	printf("  Matches[%d,%d] = %d\n", image_idx, other,
               (int) m_matches.GetNumMatches(idx));
	       // (int) m_match_lists[idx].size());

	double disti = GetCameraDistance(cameras + i, cameras + camera_idx);

	printf("  dist0, disti = %0.3e, %0.3e\n", reference_baseline, disti);

	if (disti < m_min_camera_distance_ratio * reference_baseline) {
	    printf("  Distance too low (possible panorama?)\n");
            // m_match_lists[idx].clear();
            m_matches.ClearMatch(idx);
	    continue;
	}

        std::vector<KeypointMatch> &list = m_matches.GetMatchList(idx);
	for (int j = 0; j < (int) list.size(); j++) {
	    int idx1 = list[j].m_idx1;
	    int idx2 = list[j].m_idx2;

	    int this_idx, other_idx;
	    if (image_idx == first) {
		this_idx = idx1;
		other_idx = idx2;
	    } else {
		other_idx = idx1;
		this_idx = idx2;
	    }
		
	    if (GetKey(other,other_idx).m_extra == -2) {
		/* The other key was already marked as an outlier */
		continue;
	    } else if (GetKey(image_idx,this_idx).m_extra == -2) {
		/* This key was already marked as an outlier */
		continue;
	    }

	    if (GetKey(other,other_idx).m_extra == -1 &&
		GetKey(image_idx,this_idx).m_extra >= 0) {  

		/**** Connecting an existing point *** */


		/* Connect up the other point to this one */
		int pt_idx = GetKey(image_idx,this_idx).m_extra;

		/* Check reprojection error */	    
		v2_t pr = sfm_project_final(cameras + i, points[pt_idx], 
					    true, m_estimate_distortion);

		double dx = GetKey(other,other_idx).m_x - Vx(pr);
		double dy = GetKey(other,other_idx).m_y - Vy(pr);
		    
		double proj_error = sqrt(dx * dx + dy * dy);

		if (proj_error >= 32.0) {
		    printf("  Would have connected existing match "
			   "%d ==> %d [%d] (cam: %d), \n"
			   "    but reprojection error (%0.3f) "
			   "is too high.\n", 
			   this_idx, other_idx, pt_idx, other, proj_error);
		} else {
		    printf("  Connecting existing match "
			   "%d ==> %d [%d] (cam: %d) [%0.3f]\n",
			   this_idx, other_idx, pt_idx, other, proj_error);
		    
		    GetKey(other,other_idx).m_extra = pt_idx;
		    pt_views[pt_idx].push_back(ImageKey(i, other_idx));
		}
	    } else if (GetKey(other,other_idx).m_extra == -1) {

		if (GetKey(image_idx,this_idx).m_extra != -1) {
		    printf("Error!  Key (%d,%d) shouldn't be seen yet!\n",
			   image_idx, this_idx);
		    printf("Point index is %d\n", 
			   GetKey(image_idx,this_idx).m_extra);
		}

		/* This is a new point */
		GetKey(other,other_idx).m_extra = pt_count;
		GetKey(image_idx,this_idx).m_extra = pt_count;

		/* Set up the 3D point */
		v2_t p = v2_new(GetKey(other,other_idx).m_x,
				GetKey(other,other_idx).m_y);
		    
		v2_t q = v2_new(GetKey(image_idx,this_idx).m_x,
				GetKey(image_idx,this_idx).m_y);

                if (m_optimize_for_fisheye) {
                    double p_x = Vx(p), p_y = Vy(p);
                    double q_x = Vx(q), q_y = Vy(q);
                    
                    m_image_data[other].
                        UndistortPoint(p_x, p_y, Vx(p), Vy(p));
                    m_image_data[image_idx].
                        UndistortPoint(q_x, q_y, Vx(q), Vy(q));
                }

		double proj_error = 0.0;
		bool in_front = false;
		double angle = 0.0;

		points[pt_count] = 
		    Triangulate(p, q, cameras[i], cameras[camera_idx], 
				proj_error, in_front, angle, true);


		/* Check that the angle between the rays is large
		 * enough */
		if (RAD2DEG(angle) < m_ray_angle_threshold) {
		    printf(" Ray angle %d => %d is too small (%0.3f)\n", 
			   this_idx, other_idx, RAD2DEG(angle));

		    /* Remove point */
		    GetKey(other,other_idx).m_extra = -1;
		    GetKey(image_idx,this_idx).m_extra = -1;

		    continue;
		}

		/* Check the reprojection error */
		if (proj_error >= ADD_REPROJECTION_ERROR) {
		    printf("  Projection error for %d => %d is %0.3e, "
			   "removing\n",
			   this_idx, other_idx, proj_error);

		    /* Remove point */
		    GetKey(other,other_idx).m_extra = -2;
		    GetKey(image_idx,this_idx).m_extra = -2;

		    continue;
		}

		/* Check cheirality */
		if (!in_front) {
		    printf("  Cheirality violated!\n");

		    /* Remove point */
		    GetKey(other,other_idx).m_extra = -2;
		    GetKey(image_idx,this_idx).m_extra = -2;

		    continue;
		}

		printf("  Adding match %d ==> %d [%d] (cam: %d ==> %d) "
		       "[%0.3f, %0.3f]\n", 
		       other_idx, this_idx, pt_count, image_idx, other, 
		       RAD2DEG(angle), proj_error);

		/* Finally, add the point */
		unsigned char r = GetKey(other,other_idx).m_r;
		unsigned char g = GetKey(other,other_idx).m_g;
		unsigned char b = GetKey(other,other_idx).m_b;

		colors[pt_count] = v3_new((double) r, 
					  (double) g,
					  (double) b);
    
		ImageKeyVector views;
		views.push_back(ImageKey(i, other_idx));
		views.push_back(ImageKey(camera_idx, this_idx));
		pt_views.push_back(views);

		pt_count++;

	    } else if (GetKey(other,other_idx).m_extra >= 0 && 
		       GetKey(image_idx,this_idx).m_extra == -1) {

		/* We didn't connect this point originally --
		 * check if it's now a good idea to add it in */

		/* Connect up the other point to this one */
		int pt_idx = GetKey(other,other_idx).m_extra;

		/* Check reprojection error */
		v2_t pr = sfm_project_final(cameras + camera_idx, 
					    points[pt_idx],
					    true, m_estimate_distortion);

		double dx = GetKey(image_idx,this_idx).m_x - Vx(pr);
		double dy = GetKey(image_idx,this_idx).m_y - Vy(pr);
		    
		double proj_error = sqrt(dx * dx + dy * dy);

		if (proj_error <= INIT_REPROJECTION_ERROR) {
		    printf("  Reconnecting point [%d] (%d) (error: %0.3f)\n", 
			   pt_idx, this_idx, proj_error);
		    GetKey(image_idx,this_idx).m_extra = pt_idx;
		    pt_views[pt_idx].push_back(ImageKey(camera_idx,this_idx));
		} else {
		    /* Throw out this point as an outlier */
		    GetKey(image_idx,this_idx).m_extra = -2;
		}
	    }
	}

        // m_match_lists[idx].clear();
        m_matches.ClearMatch(idx);
    }

    return pt_count;
}
/* Triangulate a subtrack */
v3_t BundlerApp::TriangulateNViews(const ImageKeyVector &views, 
                                   int *added_order, 
                                   camera_params_t *cameras,
                                   double &error, 
                                   bool explicit_camera_centers)
{
    int num_views = (int) views.size();

    v2_t *pv = new v2_t[num_views];
    double *Rs = new double[9 * num_views];
    double *ts = new double[3 * num_views];
	
    for (int i = 0; i < num_views; i++) {
	camera_params_t *cam = NULL;

	int camera_idx = views[i].first;
	int image_idx = added_order[camera_idx];
	int key_idx = views[i].second;
	Keypoint &key = GetKey(image_idx, key_idx);

	double p3[3] = { key.m_x, key.m_y, 1.0 };

        if (m_optimize_for_fisheye) {
            /* Undistort the point */
            double x = p3[0], y = p3[1];
            m_image_data[image_idx].UndistortPoint(x, y, p3[0], p3[1]);
        }

	double K[9], Kinv[9];
	GetIntrinsics(cameras[camera_idx], K);
	matrix_invert(3, K, Kinv);

	double p_n[3];
	matrix_product(3, 3, 3, 1, Kinv, p3, p_n);

        // EDIT!!!
	pv[i] = v2_new(-p_n[0], -p_n[1]);
        pv[i] = UndistortNormalizedPoint(pv[i], cameras[camera_idx]);

	cam = cameras + camera_idx;

	memcpy(Rs + 9 * i, cam->R, 9 * sizeof(double));
	if (!explicit_camera_centers) {
	    memcpy(ts + 3 * i, cam->t, 3 * sizeof(double));
	} else {
	    matrix_product(3, 3, 3, 1, cam->R, cam->t, ts + 3 * i);
	    matrix_scale(3, 1, ts + 3 * i, -1.0, ts + 3 * i);
	}
    }
    
    v3_t pt = triangulate_n(num_views, pv, Rs, ts, &error);

    error = 0.0;
    for (int i = 0; i < num_views; i++) {
	int camera_idx = views[i].first;
	int image_idx = added_order[camera_idx];
	int key_idx = views[i].second;
	Keypoint &key = GetKey(image_idx, key_idx);

	v2_t pr = sfm_project_final(cameras + camera_idx, pt, 
				    explicit_camera_centers ? 1 : 0,
                                    m_estimate_distortion ? 1 : 0);
        
        if (m_optimize_for_fisheye) {
            double x = Vx(pr), y = Vy(pr);
            m_image_data[image_idx].DistortPoint(x, y, Vx(pr), Vy(pr));
        }        

	double dx = Vx(pr) - key.m_x;
	double dy = Vy(pr) - key.m_y;

	error += dx * dx + dy * dy;
    }

    error = sqrt(error / num_views);

    delete [] pv;
    delete [] Rs;
    delete [] ts;

    return pt;
}
/* Add new points to the bundle adjustment */
int 
BundlerApp::BundleAdjustAddAllNewPoints(int num_points, int num_cameras,
                                        int *added_order,
                                        camera_params_t *cameras,
                                        v3_t *points, v3_t *colors,
                                        double reference_baseline,
                                        std::vector<ImageKeyVector> &pt_views,
                                        double max_reprojection_error,
                                        int min_views)
{
    std::vector<int> track_idxs;
    std::vector<ImageKeyVector> new_tracks;

    int num_tracks_total = (int) m_track_data.size();
    int *tracks_seen = new int[num_tracks_total];
    for (int i = 0; i < num_tracks_total; i++) {
	tracks_seen[i] = -1;
    }

    /* Gather up the projections of all the new tracks */
    for (int i = 0; i < num_cameras; i++) {
	int image_idx1 = added_order[i];

	int num_keys = GetNumKeys(image_idx1);
	
	for (int j = 0; j < num_keys; j++) {
	    Keypoint &key = GetKey(image_idx1, j);

	    if (key.m_track == -1)
		continue;  /* Key belongs to no track */

	    if (key.m_extra != -1)
		continue;  /* Key is outlier or has already been added */

	    int track_idx = key.m_track;
	    
	    /* Check if this track is already associated with a point */
	    if (m_track_data[track_idx].m_extra != -1)
		continue;

	    /* Check if we've seen this track */
	    int seen = tracks_seen[track_idx];

	    if (seen == -1) {
		/* We haven't yet seen this track, create a new track */
		tracks_seen[track_idx] = (int) new_tracks.size();

		ImageKeyVector track;
		track.push_back(ImageKey(i, j));
		new_tracks.push_back(track);
		track_idxs.push_back(track_idx);
	    } else {
		new_tracks[seen].push_back(ImageKey(i, j));
	    }
	}
    }
    
    delete [] tracks_seen;

    /* Now for each (sub) track, triangulate to see if the track is
     * consistent */
    int pt_count = num_points;

    int num_ill_conditioned = 0;
    int num_high_reprojection = 0;
    int num_cheirality_failed = 0;
    int num_added = 0;

    int num_tracks = (int) new_tracks.size();
    for (int i = 0; i < num_tracks; i++) {
	int num_views = (int) new_tracks[i].size();
	
	if (num_views < min_views) continue;  /* Not enough views */

#if 0
	printf("Triangulating track ");
	PrintTrack(new_tracks[i]);
	printf("\n");
#endif

	/* Check if at least two cameras fix the position of the point */
	bool conditioned = false;
	bool good_distance = false;
	double max_angle = 0.0;
	for (int j = 0; j < num_views; j++) {
	    for (int k = j+1; k < num_views; k++) {
		int camera_idx1 = new_tracks[i][j].first;
		int image_idx1 = added_order[camera_idx1];
		int key_idx1 = new_tracks[i][j].second;

		int camera_idx2 = new_tracks[i][k].first;
		int image_idx2 = added_order[camera_idx2];
		int key_idx2 = new_tracks[i][k].second;

		Keypoint &key1 = GetKey(image_idx1, key_idx1);
		Keypoint &key2 = GetKey(image_idx2, key_idx2);

		v2_t p = v2_new(key1.m_x, key1.m_y);
		v2_t q = v2_new(key2.m_x, key2.m_y);

                if (m_optimize_for_fisheye) {
                    double p_x = Vx(p), p_y = Vy(p);
                    double q_x = Vx(q), q_y = Vy(q);
                    
                    m_image_data[image_idx1].
                        UndistortPoint(p_x, p_y, Vx(p), Vy(p));
                    m_image_data[image_idx2].
                        UndistortPoint(q_x, q_y, Vx(q), Vy(q));
                }

		double angle = ComputeRayAngle(p, q, 
					       cameras[camera_idx1], 
					       cameras[camera_idx2]);

		if (angle > max_angle)
		    max_angle = angle;

		/* Check that the angle between the rays is large
		 * enough */
		if (RAD2DEG(angle) >= m_ray_angle_threshold) {
		    conditioned = true;
		}

#if 0
		double dist_jk = 
		    GetCameraDistance(cameras + j, cameras + k, 
				      m_explicit_camera_centers);

		if (dist_jk > m_min_camera_distance_ratio * reference_baseline)
		    good_distance = true;
#else
                good_distance = true;
#endif
	    }
	}
	
	if (!conditioned || !good_distance) {
	    num_ill_conditioned++;

#if 0
	    printf(">> Track is ill-conditioned [max_angle = %0.3f]\n", 
		   RAD2DEG(max_angle));
	    fflush(stdout);
#endif
	    continue;
	}
	
	double error;
	v3_t pt;

        if (!m_panorama_mode) {
            pt = TriangulateNViews(new_tracks[i], added_order, cameras, 
                                   error, true);
        } else {
            pt = GeneratePointAtInfinity(new_tracks[i], added_order, cameras, 
                                         error, true);
        }
       
		// Changed by Wan, Yi
	if (::isnan(error) || error > max_reprojection_error) {
	    num_high_reprojection++;
#if 0
	    printf(">> Reprojection error [%0.3f] is too large\n", error);
	    fflush(stdout);
#endif
	    continue;	    
	}

	bool all_in_front = true;
	for (int j = 0; j < num_views; j++) {
	    int camera_idx = new_tracks[i][j].first;
	    bool in_front = CheckCheirality(pt, cameras[camera_idx]);
	 
	    if (!in_front) {
		all_in_front = false;
		break;
	    }
	}

	if (!all_in_front) {
	    num_cheirality_failed++;

#if 0
	    printf(">> Cheirality check failed\n");
	    fflush(stdout);
#endif
	    continue;
	}
	
	/* All tests succeeded, so let's add the point */
#if 0
	printf("Triangulating track ");
	PrintTrack(new_tracks[i]);
	printf("\n");
	printf(">> All tests succeeded [%0.3f, %0.3f] for point [%d]\n", 
	       RAD2DEG(max_angle), error, pt_count);
#endif

	fflush(stdout);

	points[pt_count] = pt;

	int camera_idx = new_tracks[i][0].first;
	int image_idx = added_order[camera_idx];
	int key_idx = new_tracks[i][0].second;

	unsigned char r = GetKey(image_idx, key_idx).m_r;
	unsigned char g = GetKey(image_idx, key_idx).m_g;
	unsigned char b = GetKey(image_idx, key_idx).m_b;
	colors[pt_count] = v3_new((double) r, (double) g, (double) b);
    
	pt_views.push_back(new_tracks[i]);

	/* Set the point index on the keys */
	for (int j = 0; j < num_views; j++) {
	    int camera_idx = new_tracks[i][j].first;
	    int image_idx = added_order[camera_idx];
	    int key_idx = new_tracks[i][j].second;
	    GetKey(image_idx, key_idx).m_extra = pt_count;
	}

	int track_idx = track_idxs[i];
	m_track_data[track_idx].m_extra = pt_count;
	
	pt_count++;
        num_added++;
    }

    printf("[AddAllNewPoints] Added %d new points\n", num_added);
    printf("[AddAllNewPoints] Ill-conditioned tracks: %d\n", 
           num_ill_conditioned);
    printf("[AddAllNewPoints] Bad reprojections: %d\n", num_high_reprojection);
    printf("[AddAllNewPoints] Failed cheirality checks: %d\n", 
           num_cheirality_failed);

    return pt_count;
}
Exemple #5
0
/* Dump an output file containing information about the current
* state of the world */
void BaseApp::DumpOutputFile(const char *output_dir, const char *filename, 
                             int num_images, int num_cameras, int num_points,
                             int *added_order, 
                             camera_params_t *cameras, 
                             v3_t *points, v3_t *colors,
                             std::vector<ImageKeyVector> &pt_views
                             /*bool output_radial_distortion*/)
{
    clock_t start = clock();

    int num_visible_points = 0;
    
    for (int i = 0; i < num_points; i++) {
        if (pt_views[i].size() > 0)
            num_visible_points++;
    }

    char buf[256];
    sprintf(buf, "%s/%s", output_dir, filename);

    FILE *f = fopen(buf, "w");
    if (f == NULL) {
        printf("Error opening file %s for writing\n", buf);
        return;
    }

    // if (output_radial_distortion) {
    /* Print version number */
    // fprintf(f, "# Bundle file v0.4\n");
    fprintf(f, "# Bundle file v0.3\n");
    // }

    fprintf(f, "%d %d\n", num_images, num_visible_points);

    /* Dump cameras */
    for (int i = 0; i < num_images; i++) {

#if 0
        /* Print the name of the file */
        fprintf(f, "%s %d %d\n", 
                m_image_data[i].m_name, 
                m_image_data[i].GetWidth(), m_image_data[i].GetHeight());
#endif

        int idx = -1;
        for (int j = 0; j < num_cameras; j++) {
            if (added_order[j] == i) {
                idx = j;
                break;
            }
        }

        if (idx == -1) {
            // if (!output_radial_distortion)
            //     fprintf(f, "0\n");
            // else
            fprintf(f, "0 0 0\n");
            fprintf(f, "0 0 0\n0 0 0\n0 0 0\n0 0 0\n");
        } else {
            // if (!output_radial_distortion)
            //     fprintf(f, "%0.10e\n", cameras[idx].f);
            // else
            fprintf(f, "%0.10e %0.10e %0.10e\n", 
                    cameras[idx].f, cameras[idx].k[0], cameras[idx].k[1]);

            fprintf(f, "%0.10e %0.10e %0.10e\n", 
                cameras[idx].R[0], 
                cameras[idx].R[1], 
                cameras[idx].R[2]);
            fprintf(f, "%0.10e %0.10e %0.10e\n", 
                cameras[idx].R[3], 
                cameras[idx].R[4], 
                cameras[idx].R[5]);
            fprintf(f, "%0.10e %0.10e %0.10e\n", 
                cameras[idx].R[6], 
                cameras[idx].R[7], 
                cameras[idx].R[8]);

            double t[3];
            matrix_product(3, 3, 3, 1, cameras[idx].R, cameras[idx].t, t);
            matrix_scale(3, 1, t, -1.0, t);
            fprintf(f, "%0.10e %0.10e %0.10e\n", t[0], t[1], t[2]);
        }
    }

    /* Dump points */
    for (int i = 0; i < num_points; i++) {
        int num_visible = (int) pt_views[i].size();

        if (num_visible > 0) {

            /* Position */
            fprintf(f, "%0.10e %0.10e %0.10e\n", 
                    Vx(points[i]), Vy(points[i]), Vz(points[i]));
            // Vx(points[idx]), Vy(points[idx]), Vz(points[idx]));

            /* Color */
            fprintf(f, "%d %d %d\n", 
                    iround(Vx(colors[i])), 
                    iround(Vy(colors[i])), 
                    iround(Vz(colors[i])));

            int num_visible = (int) pt_views[i].size();
            fprintf(f, "%d", num_visible);
            for (int j = 0; j < num_visible; j++) {
                int img = added_order[pt_views[i][j].first];
                int key = pt_views[i][j].second;
                
                double x = m_image_data[img].m_keys[key].m_x;
                double y = m_image_data[img].m_keys[key].m_y;
                
                fprintf(f, " %d %d %0.4f %0.4f", img, key, x, y);
            }
            
            fprintf(f, "\n");
        }
    }

#if 0
    /* Finally, dump all outliers */
    ImageKeyVector outliers;
    for (int i = 0; i < num_images; i++) {
        /* Find the index of this camera in the ordering */
        int idx = -1;
        for (int j = 0; j < num_cameras; j++) {
            if (added_order[j] == i) {
                idx = j;
                break;
            }
        }

        if (idx == -1) continue;

        int num_keys = GetNumKeys(i);
        for (int j = 0; j < num_keys; j++) {
            if (GetKey(i,j).m_extra == -2) {
                outliers.push_back(ImageKey(i,j));
            }
        }
    }

    int num_outliers = (int) outliers.size();
    fprintf(f, "%d\n", num_outliers);

    for (int i = 0; i < num_outliers; i++) {
        fprintf(f, "%d %d\n", outliers[i].first, outliers[i].second);
    }
#endif

    fclose(f);

    clock_t end = clock();

    printf("[DumpOutputFile] Wrote file in %0.3fs\n",
        (double) (end - start) / (double) CLOCKS_PER_SEC);
}