size_t BasicHDT::loadMMap(unsigned char *ptr, unsigned char *ptrMax, ProgressListener *listener) { size_t count=0; ControlInformation controlInformation; IntermediateListener iListener(listener); // Load Global ControlInformation count+=controlInformation.load(&ptr[count], ptrMax); std::string hdtFormat = controlInformation.getFormat(); if(hdtFormat!=HDTVocabulary::HDT_CONTAINER) { throw "This software cannot open this version of HDT File."; } // Load Header iListener.setRange(0,5); controlInformation.load(&ptr[count], ptrMax); delete header; header = HDTFactory::readHeader(controlInformation); count+= header->load(&ptr[count], ptrMax, &iListener); // Load dictionary iListener.setRange(5, 60); controlInformation.load(&ptr[count], ptrMax); delete dictionary; dictionary = HDTFactory::readDictionary(controlInformation); count += dictionary->load(&ptr[count], ptrMax, &iListener); // Load triples iListener.setRange(60,100); controlInformation.load(&ptr[count], ptrMax); delete triples; triples = HDTFactory::readTriples(controlInformation); count += triples->load(&ptr[count], ptrMax, &iListener); return count; }
size_t PlainHeader::load(unsigned char *ptr, unsigned char *ptrMax, ProgressListener *listener) { size_t count = 0; // Read ControlInformation ControlInformation controlInformation; count += controlInformation.load(&ptr[count], ptrMax); std::string format = controlInformation.getFormat(); uint32_t headerSize = controlInformation.getUint("length"); // FIXME: Use format to create custom parser. if(format!=HDTVocabulary::HEADER_NTRIPLES) { throw "This Header format is not supported"; } string str(&ptr[count], &ptr[count+headerSize]); // Convert into a stringstream stringstream strstream(str, stringstream::in); triples.clear(); // Parse header RDFParserNtriples parser(strstream, NTRIPLES); while(parser.hasNext()) { TripleString *ts = parser.next(); triples.push_back(*ts); } count+=headerSize; return count; }
void BasicHDT::loadOrCreateIndex(ProgressListener *listener) { string indexname = this->fileName + ".index"; ifstream in(indexname.c_str(), ios::binary); if(in.good()) { if(mappedHDT) { // Map this->loadMMapIndex(listener); } else { // Read from file ControlInformation ci; ci.load(in); triples->loadIndex(in, ci, listener); } in.close(); } else { IntermediateListener iListener(listener); iListener.setRange(0,90); triples->generateIndex(&iListener); iListener.setRange(90,100); this->saveIndex(&iListener); } }
size_t FourSectionDictionary::load(unsigned char *ptr, unsigned char *ptrMax, ProgressListener *listener) { size_t count=0; IntermediateListener iListener(listener); ControlInformation ci; count += ci.load(&ptr[count], ptrMax); //this->mapping = ci.getUint("mapping"); this->mapping = MAPPING2; this->sizeStrings = ci.getUint("sizeStrings"); iListener.setRange(0,25); iListener.notifyProgress(0, "Dictionary read shared area."); delete shared; shared = csd::CSD::create(ptr[count]); if(shared==NULL){ shared = new csd::CSD_PFC(); throw std::runtime_error("Could not read shared."); } count += shared->load(&ptr[count], ptrMax); //shared = new csd::CSD_Cache(shared); iListener.setRange(25,50); iListener.notifyProgress(0, "Dictionary read subjects."); delete subjects; subjects = csd::CSD::create(ptr[count]); if(subjects==NULL){ subjects = new csd::CSD_PFC(); throw std::runtime_error("Could not read subjects."); } count += subjects->load(&ptr[count], ptrMax); //subjects = new csd::CSD_Cache(subjects); iListener.setRange(50,75); iListener.notifyProgress(0, "Dictionary read predicates."); delete predicates; predicates = csd::CSD::create(ptr[count]); if(predicates==NULL){ predicates = new csd::CSD_PFC(); throw std::runtime_error("Could not read predicates."); } count += predicates->load(&ptr[count], ptrMax); predicates = new csd::CSD_Cache2(predicates); iListener.setRange(75,100); iListener.notifyProgress(0, "Dictionary read objects."); delete objects; objects = csd::CSD::create(ptr[count]); if(objects==NULL){ objects = new csd::CSD_PFC(); throw std::runtime_error("Could not read objects."); } count += objects->load(&ptr[count], ptrMax); //objects = new csd::CSD_Cache(objects); return count; }
void BasicHDT::loadFromHDT(std::istream & input, ProgressListener *listener) { try { ControlInformation controlInformation; IntermediateListener iListener(listener); // Load Global ControlInformation. controlInformation.load(input); std::string hdtFormat = controlInformation.getFormat(); if(hdtFormat!=HDTVocabulary::HDT_CONTAINER) { throw "This software cannot open this version of HDT File."; } // Load header iListener.setRange(0,5); controlInformation.load(input); delete header; header = HDTFactory::readHeader(controlInformation); header->load(input, controlInformation, &iListener); //Load Dictionary. iListener.setRange(5, 60); controlInformation.load(input); delete dictionary; dictionary = HDTFactory::readDictionary(controlInformation); dictionary->load(input, controlInformation, &iListener); // Load Triples iListener.setRange(60,100); controlInformation.load(input); delete triples; triples = HDTFactory::readTriples(controlInformation); triples->load(input, controlInformation, &iListener); } catch (const char *ex) { cout << "Exception loading HDT: " << ex; deleteComponents(); createComponents(); throw ex; } catch (char *ex) { cout << "Exception loading HDT: " << ex; deleteComponents(); createComponents(); throw ex; } }
int main(int argc, char **argv) { int c; string outputFile; while( (c = getopt(argc,argv,"ho:"))!=-1) { switch(c) { case 'h': help(); break; case 'o': outputFile = optarg; break; default: cout << "ERROR: Unknown option" << endl; help(); return 1; } } if(argc-optind<1) { cout << "ERROR: You must supply an HDT File" << endl << endl; help(); return 1; } try { #ifdef HAVE_LIBZ igzstream *inGz=NULL; #endif ifstream *inF=NULL; istream *in=NULL; string inputFile = argv[optind]; std::string suffix = inputFile.substr(inputFile.find_last_of(".") + 1); std::string pipeCommand; if( suffix == "gz"){ #ifdef HAVE_LIBZ in = inGz = new igzstream(inputFile.c_str()); #else throw std::runtime_error("Support for GZIP was not compiled in this version. Please Decompress the file before importing it."); #endif } else { in = inF = new ifstream(inputFile.c_str(), ios::binary); } if (!in->good()) { cerr << "Error opening file " << inputFile << endl; throw std::runtime_error("Error opening file for reading"); } ControlInformation controlInformation; // Load Global Control Information controlInformation.load(*in); // Load header controlInformation.load(*in); Header *header = HDTFactory::readHeader(controlInformation); header->load(*in, controlInformation); if( suffix == "gz") { #ifdef HAVE_LIBZ inGz->close(); #endif } else { inF->close(); } // Save IteratorTripleString *it = header->search("","",""); while(it->hasNext()) { TripleString *ts = it->next(); cout << *ts << " ."<< endl; } if(outputFile!="") { ofstream out(outputFile.c_str()); if(!out.good()){ throw std::runtime_error("Could not open output file."); } RDFSerializerNTriples serializer(out, NTRIPLES); serializer.serialize(it); out.close(); } else { RDFSerializerNTriples serializer(cout, NTRIPLES); serializer.serialize(it); } delete it; delete header; } catch (std::exception& e) { cerr << "ERROR: " << e.what() << endl; } }