void processDiagLog( const char * file ) {
Connection c;
MemoryMappedFile f;
long length;
unsigned long long L = 0;
char * root = (char*)f.map( file , L, MemoryMappedFile::SEQUENTIAL );
verify( L < 0x80000000 );
length = (long) L;
verify( root );
verify( length > 0 );
char * pos = root;
long read = 0;
while ( read < length ) {
Message m(pos,false);
int len = m.header()->len;
DbMessage d(m);
cout << len << " " << d.getns() << endl;
processMessage( c , m );
read += len;
pos += len;
}
f.close();
}
Status TripleBitBuilder::resolveTriples(TempFile& rawFacts, TempFile& facts) {
cerr<<"Sort by Subject"<<endl;
ID subjectID, objectID, predicateID;
ID lastSubject = 0, lastObject = 0, lastPredicate = 0;
unsigned count0 = 0, count1 = 0;
TempFile sortedBySubject("./SortByS"), sortedByObject("./SortByO");
Sorter::sort(rawFacts, sortedBySubject, skipIdIdId, compare123);
{
//insert into chunk
sortedBySubject.close();
MemoryMappedFile mappedIn;
assert(mappedIn.open(sortedBySubject.getFile().c_str()));
const char* reader = mappedIn.getBegin(), *limit = mappedIn.getEnd();
loadTriple(reader, subjectID, predicateID, objectID);
lastSubject = subjectID; lastPredicate = predicateID; lastObject = objectID;
reader = skipIdIdId(reader);
bool v = generateXY(subjectID, objectID);
bitmap->insertTriple(predicateID, subjectID, objectID, v, 0);
count0 = count1 = 1;
while (reader < limit) {
loadTriple(reader, subjectID, predicateID, objectID);
if(lastSubject == subjectID && lastPredicate == predicateID && lastObject == objectID) {
reader = skipIdIdId(reader);
continue;
}
if ( subjectID != lastSubject ) {
((OneConstantStatisticsBuffer*)statBuffer[0])->addStatis(lastSubject, count0);
statBuffer[2]->addStatis(lastSubject, lastPredicate, count1);
lastPredicate = predicateID;
lastSubject = subjectID;
lastObject = objectID;
count0 = count1 = 1;
} else if ( predicateID != lastPredicate ) {
statBuffer[2]->addStatis(lastSubject, lastPredicate, count1);
lastPredicate = predicateID;
lastObject = objectID;
count0++; count1 = 1;
}else {
count0++; count1++;
lastObject = objectID;
}
reader = reader + 12;
v = generateXY(subjectID, objectID);
//0 indicate the triple is sorted by subjects' id;
bitmap->insertTriple(predicateID, subjectID, objectID, v, 0);
}
mappedIn.close();
}
bitmap->flush();
((OneConstantStatisticsBuffer*)statBuffer[0])->flush();
((TwoConstantStatisticsBuffer*)statBuffer[2])->flush();
//sort
cerr << "Sort by Object" << endl;
Sorter::sort(rawFacts, sortedByObject, skipIdIdId, compare321);
{
//insert into chunk
sortedByObject.close();
MemoryMappedFile mappedIn;
assert(mappedIn.open(sortedByObject.getFile().c_str()));
const char* reader = mappedIn.getBegin(), *limit = mappedIn.getEnd();
loadTriple(reader, subjectID, predicateID, objectID);
lastSubject = subjectID; lastPredicate = predicateID; lastObject = objectID;
reader = skipIdIdId(reader);
bool v = generateXY(objectID, subjectID);
bitmap->insertTriple(predicateID, objectID, subjectID, v, 1);
count0 = count1 = 1;
while (reader < limit) {
loadTriple(reader, subjectID, predicateID, objectID);
if(lastSubject == subjectID && lastPredicate == predicateID && lastObject == objectID) {
reader = skipIdIdId(reader);
continue;
}
if ( objectID != lastObject ) {
((OneConstantStatisticsBuffer*)statBuffer[1])->addStatis(lastObject, count0);
statBuffer[3]->addStatis(lastObject, lastPredicate, count1);
lastPredicate = predicateID;
lastObject = objectID;
lastSubject = subjectID;
count0 = count1 = 1;
} else if ( predicateID != lastPredicate ) {
statBuffer[3]->addStatis(lastObject, lastPredicate, count1);
lastPredicate = predicateID;
lastSubject = subjectID;
count0++; count1 = 1;
} else {
lastSubject = subjectID;
count0++; count1++;
}
reader = skipIdIdId(reader);
v = generateXY(objectID, subjectID);
// 1 indicate the triple is sorted by objects' id;
bitmap->insertTriple(predicateID, objectID, subjectID, v, 1);
}
mappedIn.close();
}
bitmap->flush();
((OneConstantStatisticsBuffer*)statBuffer[1])->flush();
((TwoConstantStatisticsBuffer*)statBuffer[3])->flush();
rawFacts.discard();
sortedByObject.discard();
sortedBySubject.discard();
return OK;
}
void close()
{
m_mmap.close();
}