int saveBufferedIndexToDisk(indexType *thisIndex){
char filename[maxFilenameLength];
int tableID = thisIndex->tableID;
int columnID = thisIndex->columnID;
sprintf(filename, "%s%d%d", indexFilename, tableID, columnID);
FILE *saveFile = fopen(filename, "wb");
if (saveFile == NULL) return Error_IndexFileCannotOpen;
indexTypeForIO *indexForIO = (indexTypeForIO*)thisIndex;
fwrite(indexForIO, sizeof(indexTypeForIO*), 1, saveFile);
int i;
int keySize = thisIndex->keySize;
int keyType = thisIndex->keyType;
nodeType *thisLeaf = thisIndex->firstLeaf;
while (thisLeaf != NULL){
fwrite(&thisLeaf->ptrNum, sizeof(thisLeaf->ptrNum), 1, saveFile);
for (i = 0; i < thisLeaf->ptrNum; i++) fwrite(thisLeaf->key[i], keySize, 1, saveFile);
for (i = 0; i < thisLeaf->ptrNum; i++)
fwrite(thisLeaf->data[i], sizeof(resultType), 1, saveFile);
thisLeaf = thisLeaf->sibling;
}
fclose(saveFile);
freeIndex(thisIndex);
return noError;
}
void MessageQueueBase::createIndex()
{
freeIndex();
messageIndex = new unsigned[numberOfMessages];
selectedMessageForReadingPosition = 0;
for(int i = 0; i < numberOfMessages; ++i)
{
messageIndex[i] = selectedMessageForReadingPosition;
selectedMessageForReadingPosition += getMessageSize() + headerSize;
}
}
MessageQueueBase::MessageQueueBase()
#ifndef TARGET_ROBOT
:
maximumSize(0x4000000), // 64 MB
reservedSize(16384)
{
buf = (char*)malloc(reservedSize + queueHeaderSize) + queueHeaderSize;
ASSERT(buf);
#else
{
#endif
}
MessageQueueBase::~MessageQueueBase()
{
freeIndex();
if(buf)
free(buf - queueHeaderSize);
if(mappedIDs)
{
delete[] mappedIDs;
delete[] mappedIDNames;
}
}
void MessageQueueBase::setSize(unsigned size, unsigned reserveForInfrastructure)
{
this->reserveForInfrastructure = reserveForInfrastructure;
size = std::min(std::numeric_limits<unsigned>::max() - queueHeaderSize, size);
#ifdef TARGET_ROBOT
ASSERT(!buf);
buf = (char*)malloc(size + queueHeaderSize) + queueHeaderSize;
ASSERT(buf);
#else
ASSERT(size >= usedSize);
if(size < reservedSize)
{
char* newBuf = (char*)realloc(buf - queueHeaderSize, size + queueHeaderSize) + queueHeaderSize;
if(newBuf)
{
buf = newBuf;
reservedSize = size;
}
}
#endif
maximumSize = size;
}
void MessageQueueBase::clear()
{
usedSize = 0;
numberOfMessages = 0;
writePosition = 0;
writingOfLastMessageFailed = false;
selectedMessageForReadingPosition = 0;
readPosition = 0;
lastMessage = 0;
numOfMappedIDs = 0;
if(mappedIDs)
{
delete[] mappedIDs;
mappedIDs = 0;
delete[] mappedIDNames;
}
freeIndex();
}
void MessageQueueBase::removeMessage(int message)
{
freeIndex();
selectedMessageForReadingPosition = 0;
int i;
for(i = 0; i < message; ++i)
selectedMessageForReadingPosition += getMessageSize() + headerSize;
usedSize = selectedMessageForReadingPosition;
for(++i; i < numberOfMessages; ++i)
{
int mlength = getMessageSize() + headerSize;
selectedMessageForReadingPosition += mlength;
memcpy(buf + usedSize, buf + selectedMessageForReadingPosition, mlength);
usedSize = selectedMessageForReadingPosition;
}
readPosition = 0;
--numberOfMessages;
selectedMessageForReadingPosition = 0;
lastMessage = 0;
}
/**
* main... :
* sets the CLI parameters
* startup operations
* kicks off the indexing
* writes the results
* frees the large data structures in use
* done!
*/
int main (int argc, char **argv) {
gchar* inFileName = NULL;
gchar* outFileName = NULL;
if (! checkCLIParams (argc, argv, &inFileName, &outFileName)) {
//failed to parse the CLI opts
return (1);
}
// count of articles found in collection
int articleCount = 0;
// array of judy arrays for the index
Pvoid_t wikiIndex[lastNgram] = {(Pvoid_t) NULL};
indexWiki (inFileName, wikiIndex, &articleCount);
writeNgramStats (wikiIndex, articleCount);
if (writeFiles) {writeIndex (wikiIndex, articleCount, outFileName);}
freeIndex (wikiIndex);
optionalPrint ("%s", "Finished!\n");
return(0);
}
int main (int argc, char **argv)
{ char str[102400];
Index S;
int k;
Tdist r;
struct stat sdata;
struct tms t1,t2;
int numQueries = 0;
if (argc != 2)
{ fprintf (stderr,"Usage: %s index-file\n",argv[0]);
exit(1);
}
fprintf (stderr,"reading index...\n");
S = loadIndex (argv[1]);
stat (argv[1],&sdata);
fprintf (stderr,"read %lli bytes\n",(long long)sdata.st_size);
numDistances = 0;
while (true)
{ Obj qry;
int siz;
bool fixed;
if (scanf ("%[0123456789-.]s",str) == 0) break;
if (!strcmp(str,"-0")) break; // -0 -> terminate
if (str[0] == '-') // negative => kNN
{ fixed = false;
if (sscanf (str+1,"%d",&k) == 0) break; // syntax error
}
else // range query
{ fixed = true;
#ifdef CONT
if (sscanf (str,"%f",&r) == 0) break; // syntax error
#else
if (sscanf (str,"%d",&r) == 0) break; // syntax error
#endif
}
if (getchar() != ',') break; // syntax error
if (scanf ("%[^\n]s",str) == 0) break; // syntax error
if (getchar() != '\n') break; // syntax error
qry = parseobj (str);
numQueries++;
if (fixed)
{ times(&t1);
siz = search(S,qry,r,true);
times(&t2);
fprintf (stderr,"%i objects found\n",siz);
}
else
{ times(&t1);
r = searchNN (S,qry,k,true);
siz = k;
times(&t2);
#ifdef CONT
fprintf (stderr,"kNNs at distance %f\n",r);
#else
fprintf (stderr,"kNNs at distance %i\n",r);
#endif
}
}
fprintf(stderr,"Total distances per query: %f\n",
numDistances/(float)numQueries);
fprintf (stderr,"freeing...\n");
freeIndex (S,true);
fprintf (stderr,"done\n");
return 0;
}
void freeSystem(systemType *thisSystem){
int i;
for (i = 0; i < thisSystem->maxBufferedIndex; i++) freeIndex(thisSystem->indexArray[i]);
free(thisSystem);
}
