void KwNode::moveAfter(DocID id)
{
assert(file_start != NULL && file_pos < file_end && docID);
while(file_index_pos < file_index_end && *((DocID*) file_index_pos) <= id) // index lookup
{
docID = *((DocID*) file_index_pos);
file_pos = file_start + *((uint64_t*) (file_index_pos + sizeof(DocID)));
file_index_pos += sizeof(DocID) + sizeof(uint64_t);
}
while(docID < id)
{
unsigned long size = read_cn(file_pos);
file_pos += size;
if(file_pos >= file_end)
{
assert(file_pos == file_end);
if(file_pos != file_end)
std::cerr << "corrupted file : " << get_index_filename(word) << std::endl;
docID = 0;
return;
}
docID += read_cn(file_pos);
}
}
void IndexFile::delete_index(Preferences *preferences, Asset *asset)
{
char index_filename[BCTEXTLEN];
char source_filename[BCTEXTLEN];
get_index_filename(source_filename,
preferences->index_directory,
index_filename,
asset->path);
//printf("IndexFile::delete_index %s %s\n", source_filename, index_filename);
remove(index_filename);
}
int IndexFile::open_file()
{
int result = 0;
get_index_filename(source_filename,
mwindow->preferences->index_directory,
index_filename,
asset->path);
if(file = fopen(index_filename, "rb"))
{
// Index file already exists.
// Get its last size without changing the status.
Asset *test_asset = new Asset;
*test_asset = *asset;
read_info(test_asset);
FileSystem fs;
if(fs.get_date(index_filename) < fs.get_date(test_asset->path))
{
// index older than source
result = 2;
fclose(file);
file = 0;
}
else
if(fs.get_size(asset->path) != test_asset->index_bytes)
{
// source file is a different size than index source file
result = 2;
fclose(file);
file = 0;
}
else
{
fseek(file, 0, SEEK_END);
file_length = ftell(file);
fseek(file, 0, SEEK_SET);
result = 0;
}
Garbage::delete_object(test_asset);
}
else
{
// doesn't exist
result = 1;
}
return result;
}
bool KwNode::init()
{
if(file_start != NULL) // If we are already initialised, don't do nothing (useful in writer thread)
return true;
std::string filename = get_index_filename(word);
std::cout << filename << std::endl;
struct stat filestatus;
stat(filename.c_str(), &filestatus);
int file = open(filename.c_str(), O_RDONLY);
if(file == -1)
{
docID = 0;
index_size = 0;
return true;
}
file_start = reinterpret_cast<char*>(mmap(0, filestatus.st_size, PROT_READ, MAP_PRIVATE, file, 0));
if(close(file) != 0)
{
std::cerr << "KwNode::init(): Unable to close the file after mmap()" << std::endl;
exit(EXIT_FAILURE);
}
if(file_start == MAP_FAILED || ! file_start)
{
perror("KwNode::init()");
exit(EXIT_FAILURE);
}
index_size = *((uint32_t*) file_start);
file_pos = file_start + sizeof(uint32_t) + (sizeof(DocID) + sizeof(uint64_t))*index_size;
file_end = file_start + filestatus.st_size;
assert(file_end - file_start > sizeof(uint32_t));
file_index_pos = file_start + sizeof(uint32_t);
file_index_end = file_pos;
while(file_index_end > file_index_pos && ! *((uint64_t*) (file_index_end - sizeof(uint64_t)))) // the index can end with non-filled fields. So we will ignore them.
{
file_index_end -= sizeof(uint64_t) + sizeof(DocID);
index_size--;
}
docID = read_cn(file_pos);
return true;
}
void flush(std::string& word, DeltaStruct* deltaStruct, SharedStruct* shared_struct)
{
/*
* On commence par voir si le fichier existe pas
* On regarde quel est le plus grand ID : Si il y a moyen on append à la fin !
* Sinon, on copie en ajoutant/modifiant à la volée l'index dans un nouveau fichier, et
* on remplace l'ancien par le nouveau
* Fonction énorme, complexe, beurk.
*/
std::string index_filename = get_index_filename(word);
QueryNode* node;
KwNode* kw_node = new KwNode(word);
DeltaStructNode* delta_node = new DeltaStructNode(deltaStruct, word);
if(! delta_node->init())
{
std::cerr << "flush(): Unable to init DeltaNode" << std::endl;
exit(EXIT_FAILURE);
}
if(! kw_node->init())
{
std::cerr << "flush(): Unable to init KwNode" << std::endl;
exit(EXIT_FAILURE);
}
DocID lastDocID = 0;
FILE* f;
bool moveTmpFile = false; // Set that to true to move the TMP_FLUSH_PATH file to the index at the end of the process
bool fullRewrite = true; // Set that to true to write a new index
#define FOPEN_ERR(path, mode) \
f = fopen(path, mode); \
if(! f) { std::cerr << "flush(): unable to open the file " << path << std::endl; exit(EXIT_FAILURE); }
if(kw_node->getDocID() == 0) // || ! kw_node->integrityCheck()) // creating a new file
{
node = delta_node;
delete kw_node;
FOPEN_ERR(index_filename.c_str(), "wb")
}
int IndexFile::create_index(Asset *asset, MainProgressBar *progress)
{
int result = 0;
this->mwindow = mwindow;
this->asset = asset;
interrupt_flag = 0;
// open the source file
File source;
if(open_source(&source)) return 1;
get_index_filename(source_filename,
mwindow->preferences->index_directory,
index_filename,
asset->path);
// Test for index in stream table of contents
if(!source.get_index(index_filename))
{
printf("IndexFile::create_index 1\n");
redraw_edits(1);
}
else
// Build index from scratch
{
asset->index_zoom = get_required_scale(&source);
// Indexes are now built for everything since it takes too long to draw
// from CDROM source.
// total length of input file
int64_t length_source = source.get_audio_length(0);
// get amount to read at a time in floats
int64_t buffersize = 65536;
char string[BCTEXTLEN];
sprintf(string, _("Creating %s."), index_filename);
progress->update_title(string);
progress->update_length(length_source);
redraw_timer->update();
// thread out index thread
IndexThread *index_thread = new IndexThread(mwindow,
this,
asset,
index_filename,
buffersize,
length_source);
index_thread->start_build();
// current sample in source file
int64_t position = 0;
int64_t fragment_size = buffersize;
int current_buffer = 0;
// pass through file once
while(position < length_source && !result)
{
if(length_source - position < fragment_size && fragment_size == buffersize) fragment_size = length_source - position;
index_thread->input_lock[current_buffer]->lock("IndexFile::create_index 1");
index_thread->input_len[current_buffer] = fragment_size;
int cancelled = progress->update(position);
if(cancelled ||
index_thread->interrupt_flag ||
interrupt_flag)
{
result = 3;
}
for(int channel = 0; !result && channel < asset->channels; channel++)
{
source.set_audio_position(position, 0);
source.set_channel(channel);
// Read from source file
if(source.read_samples(index_thread->buffer_in[current_buffer][channel],
fragment_size,
0)) result = 1;
}
// Release buffer to thread
if(!result)
{
index_thread->output_lock[current_buffer]->unlock();
current_buffer++;
if(current_buffer >= TOTAL_BUFFERS) current_buffer = 0;
position += fragment_size;
}
else
{
index_thread->input_lock[current_buffer]->unlock();
}
}
// end thread cleanly
index_thread->input_lock[current_buffer]->lock("IndexFile::create_index 2");
index_thread->last_buffer[current_buffer] = 1;
index_thread->output_lock[current_buffer]->unlock();
index_thread->stop_build();
delete index_thread;
}
source.close_file();
open_index(asset);
close_index();
mwindow->edl->set_index_file(asset);
return 0;
}
