Status TripleBitBuilder::resolveTriples(TempFile& rawFacts, TempFile& facts) {
cerr<<"Sort by Subject"<<endl;
ID subjectID, objectID, predicateID;
ID lastSubject = 0, lastObject = 0, lastPredicate = 0;
unsigned count0 = 0, count1 = 0;
TempFile sortedBySubject("./SortByS"), sortedByObject("./SortByO");
Sorter::sort(rawFacts, sortedBySubject, skipIdIdId, compare123);
{
//insert into chunk
sortedBySubject.close();
MemoryMappedFile mappedIn;
assert(mappedIn.open(sortedBySubject.getFile().c_str()));
const char* reader = mappedIn.getBegin(), *limit = mappedIn.getEnd();
loadTriple(reader, subjectID, predicateID, objectID);
lastSubject = subjectID; lastPredicate = predicateID; lastObject = objectID;
reader = skipIdIdId(reader);
bool v = generateXY(subjectID, objectID);
bitmap->insertTriple(predicateID, subjectID, objectID, v, 0);
count0 = count1 = 1;
while (reader < limit) {
loadTriple(reader, subjectID, predicateID, objectID);
if(lastSubject == subjectID && lastPredicate == predicateID && lastObject == objectID) {
reader = skipIdIdId(reader);
continue;
}
if ( subjectID != lastSubject ) {
((OneConstantStatisticsBuffer*)statBuffer[0])->addStatis(lastSubject, count0);
statBuffer[2]->addStatis(lastSubject, lastPredicate, count1);
lastPredicate = predicateID;
lastSubject = subjectID;
lastObject = objectID;
count0 = count1 = 1;
} else if ( predicateID != lastPredicate ) {
statBuffer[2]->addStatis(lastSubject, lastPredicate, count1);
lastPredicate = predicateID;
lastObject = objectID;
count0++; count1 = 1;
}else {
count0++; count1++;
lastObject = objectID;
}
reader = reader + 12;
v = generateXY(subjectID, objectID);
//0 indicate the triple is sorted by subjects' id;
bitmap->insertTriple(predicateID, subjectID, objectID, v, 0);
}
mappedIn.close();
}
bitmap->flush();
((OneConstantStatisticsBuffer*)statBuffer[0])->flush();
((TwoConstantStatisticsBuffer*)statBuffer[2])->flush();
//sort
cerr << "Sort by Object" << endl;
Sorter::sort(rawFacts, sortedByObject, skipIdIdId, compare321);
{
//insert into chunk
sortedByObject.close();
MemoryMappedFile mappedIn;
assert(mappedIn.open(sortedByObject.getFile().c_str()));
const char* reader = mappedIn.getBegin(), *limit = mappedIn.getEnd();
loadTriple(reader, subjectID, predicateID, objectID);
lastSubject = subjectID; lastPredicate = predicateID; lastObject = objectID;
reader = skipIdIdId(reader);
bool v = generateXY(objectID, subjectID);
bitmap->insertTriple(predicateID, objectID, subjectID, v, 1);
count0 = count1 = 1;
while (reader < limit) {
loadTriple(reader, subjectID, predicateID, objectID);
if(lastSubject == subjectID && lastPredicate == predicateID && lastObject == objectID) {
reader = skipIdIdId(reader);
continue;
}
if ( objectID != lastObject ) {
((OneConstantStatisticsBuffer*)statBuffer[1])->addStatis(lastObject, count0);
statBuffer[3]->addStatis(lastObject, lastPredicate, count1);
lastPredicate = predicateID;
lastObject = objectID;
lastSubject = subjectID;
count0 = count1 = 1;
} else if ( predicateID != lastPredicate ) {
statBuffer[3]->addStatis(lastObject, lastPredicate, count1);
lastPredicate = predicateID;
lastSubject = subjectID;
count0++; count1 = 1;
} else {
lastSubject = subjectID;
count0++; count1++;
}
reader = skipIdIdId(reader);
v = generateXY(objectID, subjectID);
// 1 indicate the triple is sorted by objects' id;
bitmap->insertTriple(predicateID, objectID, subjectID, v, 1);
}
mappedIn.close();
}
bitmap->flush();
((OneConstantStatisticsBuffer*)statBuffer[1])->flush();
((TwoConstantStatisticsBuffer*)statBuffer[3])->flush();
rawFacts.discard();
sortedByObject.discard();
sortedBySubject.discard();
return OK;
}
//---------------------------------------------------------------------------
static void buildDictionary(TempFile& rawStrings,TempFile& stringTable,TempFile& stringIds,map<unsigned,unsigned>& subTypes)
// Build the dictionary
{
cerr << "Building the dictionary..." << endl;
// Sort the strings to resolve duplicates
TempFile sortedStrings(rawStrings.getBaseFile());
Sorter::sort(rawStrings,sortedStrings,skipStringIdId,compareStringIdId);
rawStrings.discard();
// Build the id map and the string list
TempFile rawIdMap(rawStrings.getBaseFile()),stringList(rawStrings.getBaseFile());
{
MemoryMappedFile strings;
ensure(strings.open(sortedStrings.getFile().c_str()));
uint64_t lastId=0; unsigned lastLen=0; const char* lastStr=0; uint64_t lastType=0;
for (const char* iter=strings.getBegin(),*limit=strings.getEnd();iter!=limit;) {
// Read the entry
unsigned stringLen; const char* stringStart;
iter=TempFile::readString(iter,stringLen,stringStart);
uint64_t id,type;
iter=TempFile::readId(iter,type);
iter=TempFile::readId(iter,id);
// A new one?
if ((!lastStr)||(stringLen!=lastLen)||(memcmp(lastStr,stringStart,stringLen)!=0)||(type!=lastType)) {
stringList.writeId(id);
stringList.writeString(stringLen,stringStart);
stringList.writeId(type);
rawIdMap.writeId(id);
rawIdMap.writeId(id);
lastId=id; lastLen=stringLen; lastStr=stringStart; lastType=type;
} else {
rawIdMap.writeId(lastId);
rawIdMap.writeId(id);
}
}
}
sortedStrings.discard();
// Sort the string list
Sorter::sort(stringList,stringTable,skipIdStringId,compareId);
stringList.discard();
// Sort the ID map
TempFile idMap(rawStrings.getBaseFile());
Sorter::sort(rawIdMap,idMap,skipIdId,compareId);
rawIdMap.discard();
// Construct new ids
TempFile newIds(rawStrings.getBaseFile());
{
MemoryMappedFile in;
ensure(in.open(idMap.getFile().c_str()));
uint64_t lastId=0,newId=0;
for (const char* iter=in.getBegin(),*limit=in.getEnd();iter!=limit;) {
uint64_t firstId,currentId;
iter=TempFile::readId(iter,firstId);
iter=TempFile::readId(iter,currentId);
if (firstId!=lastId) {
++newId;
lastId=firstId;
}
newIds.writeId(currentId);
newIds.writeId(newId);
if (subTypes.count(currentId))
subTypes[currentId]=newId;
}
}
// And a final sort
Sorter::sort(newIds,stringIds,skipIdId,compareValue);
newIds.discard();
// Resolve the subtypes if necessary
if (!subTypes.empty()) {
TempFile fixedTypes(rawStrings.getBaseFile());
MemoryMappedFile in;
ensure(in.open(stringTable.getFile().c_str()));
for (const char* iter=in.getBegin(),*limit=in.getEnd();iter!=limit;) {
uint64_t id,typeInfo;
const char* value; unsigned valueLen;
iter=TempFile::readId(TempFile::readString(TempFile::readId(iter,id),valueLen,value),typeInfo);
unsigned type=typeInfo&0xFF,subType=(typeInfo>>8);
if (Type::hasSubType(static_cast<Type::ID>(type))) {
assert(subTypes.count(subType));
typeInfo=type|(subTypes[subType]<<8);
} else {
assert(subType==0);
}
fixedTypes.writeId(id);
fixedTypes.writeString(valueLen,value);
fixedTypes.writeId(typeInfo);
}
fixedTypes.close();
fixedTypes.swap(stringTable);
}
