void KwNode::moveAfter(DocID id) { assert(file_start != NULL && file_pos < file_end && docID); while(file_index_pos < file_index_end && *((DocID*) file_index_pos) <= id) // index lookup { docID = *((DocID*) file_index_pos); file_pos = file_start + *((uint64_t*) (file_index_pos + sizeof(DocID))); file_index_pos += sizeof(DocID) + sizeof(uint64_t); } while(docID < id) { unsigned long size = read_cn(file_pos); file_pos += size; if(file_pos >= file_end) { assert(file_pos == file_end); if(file_pos != file_end) std::cerr << "corrupted file : " << get_index_filename(word) << std::endl; docID = 0; return; } docID += read_cn(file_pos); } }
void IndexFile::delete_index(Preferences *preferences, Asset *asset) { char index_filename[BCTEXTLEN]; char source_filename[BCTEXTLEN]; get_index_filename(source_filename, preferences->index_directory, index_filename, asset->path); //printf("IndexFile::delete_index %s %s\n", source_filename, index_filename); remove(index_filename); }
int IndexFile::open_file() { int result = 0; get_index_filename(source_filename, mwindow->preferences->index_directory, index_filename, asset->path); if(file = fopen(index_filename, "rb")) { // Index file already exists. // Get its last size without changing the status. Asset *test_asset = new Asset; *test_asset = *asset; read_info(test_asset); FileSystem fs; if(fs.get_date(index_filename) < fs.get_date(test_asset->path)) { // index older than source result = 2; fclose(file); file = 0; } else if(fs.get_size(asset->path) != test_asset->index_bytes) { // source file is a different size than index source file result = 2; fclose(file); file = 0; } else { fseek(file, 0, SEEK_END); file_length = ftell(file); fseek(file, 0, SEEK_SET); result = 0; } Garbage::delete_object(test_asset); } else { // doesn't exist result = 1; } return result; }
bool KwNode::init() { if(file_start != NULL) // If we are already initialised, don't do nothing (useful in writer thread) return true; std::string filename = get_index_filename(word); std::cout << filename << std::endl; struct stat filestatus; stat(filename.c_str(), &filestatus); int file = open(filename.c_str(), O_RDONLY); if(file == -1) { docID = 0; index_size = 0; return true; } file_start = reinterpret_cast<char*>(mmap(0, filestatus.st_size, PROT_READ, MAP_PRIVATE, file, 0)); if(close(file) != 0) { std::cerr << "KwNode::init(): Unable to close the file after mmap()" << std::endl; exit(EXIT_FAILURE); } if(file_start == MAP_FAILED || ! file_start) { perror("KwNode::init()"); exit(EXIT_FAILURE); } index_size = *((uint32_t*) file_start); file_pos = file_start + sizeof(uint32_t) + (sizeof(DocID) + sizeof(uint64_t))*index_size; file_end = file_start + filestatus.st_size; assert(file_end - file_start > sizeof(uint32_t)); file_index_pos = file_start + sizeof(uint32_t); file_index_end = file_pos; while(file_index_end > file_index_pos && ! *((uint64_t*) (file_index_end - sizeof(uint64_t)))) // the index can end with non-filled fields. So we will ignore them. { file_index_end -= sizeof(uint64_t) + sizeof(DocID); index_size--; } docID = read_cn(file_pos); return true; }
void flush(std::string& word, DeltaStruct* deltaStruct, SharedStruct* shared_struct) { /* * On commence par voir si le fichier existe pas * On regarde quel est le plus grand ID : Si il y a moyen on append à la fin ! * Sinon, on copie en ajoutant/modifiant à la volée l'index dans un nouveau fichier, et * on remplace l'ancien par le nouveau * Fonction énorme, complexe, beurk. */ std::string index_filename = get_index_filename(word); QueryNode* node; KwNode* kw_node = new KwNode(word); DeltaStructNode* delta_node = new DeltaStructNode(deltaStruct, word); if(! delta_node->init()) { std::cerr << "flush(): Unable to init DeltaNode" << std::endl; exit(EXIT_FAILURE); } if(! kw_node->init()) { std::cerr << "flush(): Unable to init KwNode" << std::endl; exit(EXIT_FAILURE); } DocID lastDocID = 0; FILE* f; bool moveTmpFile = false; // Set that to true to move the TMP_FLUSH_PATH file to the index at the end of the process bool fullRewrite = true; // Set that to true to write a new index #define FOPEN_ERR(path, mode) \ f = fopen(path, mode); \ if(! f) { std::cerr << "flush(): unable to open the file " << path << std::endl; exit(EXIT_FAILURE); } if(kw_node->getDocID() == 0) // || ! kw_node->integrityCheck()) // creating a new file { node = delta_node; delete kw_node; FOPEN_ERR(index_filename.c_str(), "wb") }
int IndexFile::create_index(Asset *asset, MainProgressBar *progress) { int result = 0; this->mwindow = mwindow; this->asset = asset; interrupt_flag = 0; // open the source file File source; if(open_source(&source)) return 1; get_index_filename(source_filename, mwindow->preferences->index_directory, index_filename, asset->path); // Test for index in stream table of contents if(!source.get_index(index_filename)) { printf("IndexFile::create_index 1\n"); redraw_edits(1); } else // Build index from scratch { asset->index_zoom = get_required_scale(&source); // Indexes are now built for everything since it takes too long to draw // from CDROM source. // total length of input file int64_t length_source = source.get_audio_length(0); // get amount to read at a time in floats int64_t buffersize = 65536; char string[BCTEXTLEN]; sprintf(string, _("Creating %s."), index_filename); progress->update_title(string); progress->update_length(length_source); redraw_timer->update(); // thread out index thread IndexThread *index_thread = new IndexThread(mwindow, this, asset, index_filename, buffersize, length_source); index_thread->start_build(); // current sample in source file int64_t position = 0; int64_t fragment_size = buffersize; int current_buffer = 0; // pass through file once while(position < length_source && !result) { if(length_source - position < fragment_size && fragment_size == buffersize) fragment_size = length_source - position; index_thread->input_lock[current_buffer]->lock("IndexFile::create_index 1"); index_thread->input_len[current_buffer] = fragment_size; int cancelled = progress->update(position); if(cancelled || index_thread->interrupt_flag || interrupt_flag) { result = 3; } for(int channel = 0; !result && channel < asset->channels; channel++) { source.set_audio_position(position, 0); source.set_channel(channel); // Read from source file if(source.read_samples(index_thread->buffer_in[current_buffer][channel], fragment_size, 0)) result = 1; } // Release buffer to thread if(!result) { index_thread->output_lock[current_buffer]->unlock(); current_buffer++; if(current_buffer >= TOTAL_BUFFERS) current_buffer = 0; position += fragment_size; } else { index_thread->input_lock[current_buffer]->unlock(); } } // end thread cleanly index_thread->input_lock[current_buffer]->lock("IndexFile::create_index 2"); index_thread->last_buffer[current_buffer] = 1; index_thread->output_lock[current_buffer]->unlock(); index_thread->stop_build(); delete index_thread; } source.close_file(); open_index(asset); close_index(); mwindow->edl->set_index_file(asset); return 0; }