//---------------------------------------------------------------------------
void dumpFacts(ofstream& out,TempFile& rawFacts,const string& name)
// Dump the facts
{
// Sort the facts
TempFile sortedFacts(rawFacts.getBaseFile());
Sorter::sort(rawFacts,sortedFacts,skipTriple,compareTriple,true);
// Dump the facts
{
unlink("facts.sql");
ofstream out("facts.sql");
MemoryMappedFile in;
in.open(sortedFacts.getFile().c_str());
const Triple* triplesBegin=reinterpret_cast<const Triple*>(in.getBegin());
const Triple* triplesEnd=reinterpret_cast<const Triple*>(in.getEnd());
for (const Triple* iter=triplesBegin,*limit=triplesEnd; iter!=limit; ++iter)
out << (*iter).subject << "\t" << (*iter).predicate << "\t" << (*iter).object << std::endl;
}
// And write the copy statement
out << "drop schema if exists " << name << " cascade;" << endl;
out << "create schema " << name << ";" << endl;
out << "create table " << name << ".facts(subject int not null, predicate int not null, object int not null);" << endl;
out << "copy " << name << ".facts from 'facts.sql';" << endl;
// Create indices
out << "create index facts_spo on " << name << ".facts (subject, predicate, object);" << endl;
out << "create index facts_pso on " << name << ".facts (predicate, subject, object);" << endl;
out << "create index facts_pos on " << name << ".facts (predicate, object, subject);" << endl;
}
//---------------------------------------------------------------------------
static void buildDictionary(TempFile& rawStrings,TempFile& stringTable,TempFile& stringIds,map<unsigned,unsigned>& subTypes)
// Build the dictionary
{
cerr << "Building the dictionary..." << endl;
// Sort the strings to resolve duplicates
TempFile sortedStrings(rawStrings.getBaseFile());
Sorter::sort(rawStrings,sortedStrings,skipStringIdId,compareStringIdId);
rawStrings.discard();
// Build the id map and the string list
TempFile rawIdMap(rawStrings.getBaseFile()),stringList(rawStrings.getBaseFile());
{
MemoryMappedFile strings;
ensure(strings.open(sortedStrings.getFile().c_str()));
uint64_t lastId=0; unsigned lastLen=0; const char* lastStr=0; uint64_t lastType=0;
for (const char* iter=strings.getBegin(),*limit=strings.getEnd();iter!=limit;) {
// Read the entry
unsigned stringLen; const char* stringStart;
iter=TempFile::readString(iter,stringLen,stringStart);
uint64_t id,type;
iter=TempFile::readId(iter,type);
iter=TempFile::readId(iter,id);
// A new one?
if ((!lastStr)||(stringLen!=lastLen)||(memcmp(lastStr,stringStart,stringLen)!=0)||(type!=lastType)) {
stringList.writeId(id);
stringList.writeString(stringLen,stringStart);
stringList.writeId(type);
rawIdMap.writeId(id);
rawIdMap.writeId(id);
lastId=id; lastLen=stringLen; lastStr=stringStart; lastType=type;
} else {
rawIdMap.writeId(lastId);
rawIdMap.writeId(id);
}
}
}
sortedStrings.discard();
// Sort the string list
Sorter::sort(stringList,stringTable,skipIdStringId,compareId);
stringList.discard();
// Sort the ID map
TempFile idMap(rawStrings.getBaseFile());
Sorter::sort(rawIdMap,idMap,skipIdId,compareId);
rawIdMap.discard();
// Construct new ids
TempFile newIds(rawStrings.getBaseFile());
{
MemoryMappedFile in;
ensure(in.open(idMap.getFile().c_str()));
uint64_t lastId=0,newId=0;
for (const char* iter=in.getBegin(),*limit=in.getEnd();iter!=limit;) {
uint64_t firstId,currentId;
iter=TempFile::readId(iter,firstId);
iter=TempFile::readId(iter,currentId);
if (firstId!=lastId) {
++newId;
lastId=firstId;
}
newIds.writeId(currentId);
newIds.writeId(newId);
if (subTypes.count(currentId))
subTypes[currentId]=newId;
}
}
// And a final sort
Sorter::sort(newIds,stringIds,skipIdId,compareValue);
newIds.discard();
// Resolve the subtypes if necessary
if (!subTypes.empty()) {
TempFile fixedTypes(rawStrings.getBaseFile());
MemoryMappedFile in;
ensure(in.open(stringTable.getFile().c_str()));
for (const char* iter=in.getBegin(),*limit=in.getEnd();iter!=limit;) {
uint64_t id,typeInfo;
const char* value; unsigned valueLen;
iter=TempFile::readId(TempFile::readString(TempFile::readId(iter,id),valueLen,value),typeInfo);
unsigned type=typeInfo&0xFF,subType=(typeInfo>>8);
if (Type::hasSubType(static_cast<Type::ID>(type))) {
assert(subTypes.count(subType));
typeInfo=type|(subTypes[subType]<<8);
} else {
assert(subType==0);
}
fixedTypes.writeId(id);
fixedTypes.writeString(valueLen,value);
fixedTypes.writeId(typeInfo);
}
fixedTypes.close();
fixedTypes.swap(stringTable);
}
