NNIndex<Distance>* load_saved_index(const Matrix<typename Distance::ElementType>& dataset, const std::string& filename, Distance distance)
{
typedef typename Distance::ElementType ElementType;
FILE* fin = fopen(filename.c_str(), "rb");
if (fin == NULL) {
return NULL;
}
IndexHeader header = load_header(fin);
if (header.data_type != flann_datatype<ElementType>::value) {
throw FLANNException("Datatype of saved index is different than of the one to be created.");
}
// We cannot make this check now, because the index and the dataset might be of different size
// some entries of the dataset might not be used in the index
/*
if ((size_t(header.rows) != dataset.rows)||(size_t(header.cols) != dataset.cols)) {
throw FLANNException("The index saved belongs to a different dataset");
}
*/
IndexParams params;
params["algorithm"] = header.index_type;
NNIndex<Distance>* nnIndex = create_index_by_type<Distance>(dataset, params, distance);
nnIndex->loadIndex(fin);
fclose(fin);
return nnIndex;
}
IndexHeader load_header(FILE* stream)
{
IndexHeader header;
int read_size = fread(&header,sizeof(header),1,stream);
if (read_size!=1) {
throw FLANNException("Invalid index file, cannot read");
}
if (strcmp(header.signature,FLANN_SIGNATURE)!=0) {
throw FLANNException("Invalid index file, wrong signature");
}
return header;
}
/**
* Save index to file
* @param filename
*/
void save(std::string filename)
{
FILE* fout = fopen(filename.c_str(), "wb");
if (fout == NULL) {
throw FLANNException("Cannot open file");
}
nnIndex_->saveIndex(fout);
fclose(fout);
}
void save(std::string filename)
{
FILE* fout = fopen(filename.c_str(), "wb");
if (fout == NULL) {
throw FLANNException("Cannot open file: " + filename);
}
save_header(fout, *nnIndex_);
saveIndex(fout);
fclose(fout);
}
IndexType* load_saved_index(const Matrix<ElementType>& dataset, const std::string& filename, Distance distance)
{
FILE* fin = fopen(filename.c_str(), "rb");
if (fin == NULL) {
return NULL;
}
IndexHeader header = load_header(fin);
if (header.data_type != flann_datatype<ElementType>::value) {
throw FLANNException("Datatype of saved index is different than of the one to be created.");
}
if ((size_t(header.rows) != dataset.rows)||(size_t(header.cols) != dataset.cols)) {
throw FLANNException("The index saved belongs to a different dataset");
}
IndexParams params;
params["algorithm"] = header.index_type;
IndexType* nnIndex = create_index_by_type<Distance>(header.index_type, dataset, params, distance);
nnIndex->loadIndex(fin);
fclose(fin);
return nnIndex;
}
float search_with_ground_truth(NNIndex& index, const Matrix<float>& inputData, const Matrix<float>& testData, const Matrix<int>& matches, int nn, int checks, float& time, float& dist, int skipMatches)
{
if (matches.cols<nn) {
logger.info("matches.cols=%d, nn=%d\n",matches.cols,nn);
throw FLANNException("Ground truth is not computed for as many neighbors as requested");
}
KNNResultSet resultSet(nn+skipMatches);
SearchParams searchParams(checks);
int correct = 0;
float distR = 0;
StartStopTimer t;
int repeats = 0;
while (t.value<0.2) {
repeats++;
t.start();
correct = 0;
distR = 0;
for (int i = 0; i < testData.rows; i++) {
float* target = testData[i];
resultSet.init(target, testData.cols);
index.findNeighbors(resultSet,target, searchParams);
int* neighbors = resultSet.getNeighbors();
neighbors = neighbors+skipMatches;
correct += countCorrectMatches(neighbors,matches[i], nn);
distR += computeDistanceRaport(inputData, target,neighbors,matches[i], testData.cols, nn);
}
t.stop();
}
time = (float)(t.value/repeats);
float precicion = (float)correct/(nn*testData.rows);
dist = distR/(testData.rows*nn);
logger.info("%8d %10.4g %10.5g %10.5g %10.5g\n",
checks, precicion, time, 1000.0 * time / testData.rows, dist);
return precicion;
}
Index<Distance>::Index(const std::string& file_name, const std::string& dataset_name, const IndexParams& params)
{
boost::mpi::communicator world;
flann_algorithm_t index_type = params.getIndexType();
if (index_type == SAVED) {
throw FLANNException("Saving/loading of MPI indexes is not currently supported.");
}
flann::mpi::load_from_file(dataset, file_name, dataset_name);
flann_index = new flann::Index<Distance>(dataset, params);
std::vector<int> sizes;
// get the sizes of all MPI indices
all_gather(world, flann_index->size(), sizes);
size_ = 0;
offset_ = 0;
for (size_t i = 0;i < sizes.size();++i) {
if (i < world.rank()) offset_ += sizes[i];
size_ += sizes[i];
}
}