size_t BasicHDT::loadMMap(unsigned char *ptr, unsigned char *ptrMax, ProgressListener *listener) {
size_t count=0;
ControlInformation controlInformation;
IntermediateListener iListener(listener);
// Load Global ControlInformation
count+=controlInformation.load(&ptr[count], ptrMax);
std::string hdtFormat = controlInformation.getFormat();
if(hdtFormat!=HDTVocabulary::HDT_CONTAINER) {
throw "This software cannot open this version of HDT File.";
}
// Load Header
iListener.setRange(0,5);
controlInformation.load(&ptr[count], ptrMax);
delete header;
header = HDTFactory::readHeader(controlInformation);
count+= header->load(&ptr[count], ptrMax, &iListener);
// Load dictionary
iListener.setRange(5, 60);
controlInformation.load(&ptr[count], ptrMax);
delete dictionary;
dictionary = HDTFactory::readDictionary(controlInformation);
count += dictionary->load(&ptr[count], ptrMax, &iListener);
// Load triples
iListener.setRange(60,100);
controlInformation.load(&ptr[count], ptrMax);
delete triples;
triples = HDTFactory::readTriples(controlInformation);
count += triples->load(&ptr[count], ptrMax, &iListener);
return count;
}
size_t PlainHeader::load(unsigned char *ptr, unsigned char *ptrMax, ProgressListener *listener)
{
size_t count = 0;
// Read ControlInformation
ControlInformation controlInformation;
count += controlInformation.load(&ptr[count], ptrMax);
std::string format = controlInformation.getFormat();
uint32_t headerSize = controlInformation.getUint("length");
// FIXME: Use format to create custom parser.
if(format!=HDTVocabulary::HEADER_NTRIPLES) {
throw "This Header format is not supported";
}
string str(&ptr[count], &ptr[count+headerSize]);
// Convert into a stringstream
stringstream strstream(str, stringstream::in);
triples.clear();
// Parse header
RDFParserNtriples parser(strstream, NTRIPLES);
while(parser.hasNext()) {
TripleString *ts = parser.next();
triples.push_back(*ts);
}
count+=headerSize;
return count;
}
void BasicHDT::loadOrCreateIndex(ProgressListener *listener) {
string indexname = this->fileName + ".index";
ifstream in(indexname.c_str(), ios::binary);
if(in.good()) {
if(mappedHDT) {
// Map
this->loadMMapIndex(listener);
} else {
// Read from file
ControlInformation ci;
ci.load(in);
triples->loadIndex(in, ci, listener);
}
in.close();
} else {
IntermediateListener iListener(listener);
iListener.setRange(0,90);
triples->generateIndex(&iListener);
iListener.setRange(90,100);
this->saveIndex(&iListener);
}
}
size_t FourSectionDictionary::load(unsigned char *ptr, unsigned char *ptrMax, ProgressListener *listener)
{
size_t count=0;
IntermediateListener iListener(listener);
ControlInformation ci;
count += ci.load(&ptr[count], ptrMax);
//this->mapping = ci.getUint("mapping");
this->mapping = MAPPING2;
this->sizeStrings = ci.getUint("sizeStrings");
iListener.setRange(0,25);
iListener.notifyProgress(0, "Dictionary read shared area.");
delete shared;
shared = csd::CSD::create(ptr[count]);
if(shared==NULL){
shared = new csd::CSD_PFC();
throw std::runtime_error("Could not read shared.");
}
count += shared->load(&ptr[count], ptrMax);
//shared = new csd::CSD_Cache(shared);
iListener.setRange(25,50);
iListener.notifyProgress(0, "Dictionary read subjects.");
delete subjects;
subjects = csd::CSD::create(ptr[count]);
if(subjects==NULL){
subjects = new csd::CSD_PFC();
throw std::runtime_error("Could not read subjects.");
}
count += subjects->load(&ptr[count], ptrMax);
//subjects = new csd::CSD_Cache(subjects);
iListener.setRange(50,75);
iListener.notifyProgress(0, "Dictionary read predicates.");
delete predicates;
predicates = csd::CSD::create(ptr[count]);
if(predicates==NULL){
predicates = new csd::CSD_PFC();
throw std::runtime_error("Could not read predicates.");
}
count += predicates->load(&ptr[count], ptrMax);
predicates = new csd::CSD_Cache2(predicates);
iListener.setRange(75,100);
iListener.notifyProgress(0, "Dictionary read objects.");
delete objects;
objects = csd::CSD::create(ptr[count]);
if(objects==NULL){
objects = new csd::CSD_PFC();
throw std::runtime_error("Could not read objects.");
}
count += objects->load(&ptr[count], ptrMax);
//objects = new csd::CSD_Cache(objects);
return count;
}
void BasicHDT::loadFromHDT(std::istream & input, ProgressListener *listener)
{
try {
ControlInformation controlInformation;
IntermediateListener iListener(listener);
// Load Global ControlInformation.
controlInformation.load(input);
std::string hdtFormat = controlInformation.getFormat();
if(hdtFormat!=HDTVocabulary::HDT_CONTAINER) {
throw "This software cannot open this version of HDT File.";
}
// Load header
iListener.setRange(0,5);
controlInformation.load(input);
delete header;
header = HDTFactory::readHeader(controlInformation);
header->load(input, controlInformation, &iListener);
//Load Dictionary.
iListener.setRange(5, 60);
controlInformation.load(input);
delete dictionary;
dictionary = HDTFactory::readDictionary(controlInformation);
dictionary->load(input, controlInformation, &iListener);
// Load Triples
iListener.setRange(60,100);
controlInformation.load(input);
delete triples;
triples = HDTFactory::readTriples(controlInformation);
triples->load(input, controlInformation, &iListener);
} catch (const char *ex) {
cout << "Exception loading HDT: " << ex;
deleteComponents();
createComponents();
throw ex;
} catch (char *ex) {
cout << "Exception loading HDT: " << ex;
deleteComponents();
createComponents();
throw ex;
}
}
int main(int argc, char **argv) {
int c;
string outputFile;
while( (c = getopt(argc,argv,"ho:"))!=-1) {
switch(c) {
case 'h':
help();
break;
case 'o':
outputFile = optarg;
break;
default:
cout << "ERROR: Unknown option" << endl;
help();
return 1;
}
}
if(argc-optind<1) {
cout << "ERROR: You must supply an HDT File" << endl << endl;
help();
return 1;
}
try {
#ifdef HAVE_LIBZ
igzstream *inGz=NULL;
#endif
ifstream *inF=NULL;
istream *in=NULL;
string inputFile = argv[optind];
std::string suffix = inputFile.substr(inputFile.find_last_of(".") + 1);
std::string pipeCommand;
if( suffix == "gz"){
#ifdef HAVE_LIBZ
in = inGz = new igzstream(inputFile.c_str());
#else
throw std::runtime_error("Support for GZIP was not compiled in this version. Please Decompress the file before importing it.");
#endif
} else {
in = inF = new ifstream(inputFile.c_str(), ios::binary);
}
if (!in->good())
{
cerr << "Error opening file " << inputFile << endl;
throw std::runtime_error("Error opening file for reading");
}
ControlInformation controlInformation;
// Load Global Control Information
controlInformation.load(*in);
// Load header
controlInformation.load(*in);
Header *header = HDTFactory::readHeader(controlInformation);
header->load(*in, controlInformation);
if( suffix == "gz") {
#ifdef HAVE_LIBZ
inGz->close();
#endif
} else {
inF->close();
}
// Save
IteratorTripleString *it = header->search("","","");
while(it->hasNext()) {
TripleString *ts = it->next();
cout << *ts << " ."<< endl;
}
if(outputFile!="") {
ofstream out(outputFile.c_str());
if(!out.good()){
throw std::runtime_error("Could not open output file.");
}
RDFSerializerNTriples serializer(out, NTRIPLES);
serializer.serialize(it);
out.close();
} else {
RDFSerializerNTriples serializer(cout, NTRIPLES);
serializer.serialize(it);
}
delete it;
delete header;
} catch (std::exception& e) {
cerr << "ERROR: " << e.what() << endl;
}
}
