void BasicHDT::loadTriples(const char* fileName, const char* baseUri, RDFNotation notation, ProgressListener* listener) {
// Generate Triples
ModifiableTriples* triplesList = new TriplesList(spec);
//ModifiableTriples *triplesList = new TriplesKyoto(spec);
//ModifiableTriples *triplesList = new TripleListDisk();
StopWatch st;
IntermediateListener iListener(listener);
try {
NOTIFY(listener, "Loading Triples", 0, 100);
iListener.setRange(0, 60);
triplesList->startProcessing(&iListener);
TriplesLoader tripLoader(dictionary, triplesList, &iListener);
RDFParserCallback *pars = RDFParserCallback::getParserCallback(
notation);
pars->doParse(fileName, baseUri, notation, &tripLoader);
delete pars;
header->insert("_:statistics", HDTVocabulary::ORIGINAL_SIZE, tripLoader.getSize());
triplesList->stopProcessing(&iListener);
// SORT & Duplicates
TripleComponentOrder order = parseOrder(
spec.get("triplesOrder").c_str());
if (order == Unknown) {
order = SPO;
}
iListener.setRange(80, 85);
triplesList->sort(order, &iListener);
iListener.setRange(85, 90);
triplesList->removeDuplicates(&iListener);
} catch (const char *e) {
cout << "Catch exception triples" << e << endl;
delete triplesList;
throw e;
} catch (char *e) {
cout << "Catch exception triples" << e << endl;
delete triplesList;
throw e;
}
if (triples->getType() == triplesList->getType()) {
delete triples;
triples = triplesList;
} else {
iListener.setRange(90, 100);
try {
triples->load(*triplesList, &iListener);
} catch (const char* e) {
delete triplesList;
throw e;
}
delete triplesList;
}
//cout << triples->getNumberOfElements() << " triples added in " << st << endl << endl;
}
CompactTriples::CompactTriples(HDTSpecification &specification) : numTriples(0), spec(specification) {
std::string orderStr = spec.get("triplesOrder");
order= parseOrder(orderStr.c_str());
if(order==Unknown)
order = SPO;
streamY = IntSequence::getArray(spec.get("stream.y"));
streamZ = IntSequence::getArray(spec.get("stream.z"));
}
BitmapTriples::BitmapTriples(HDTSpecification &specification) : spec(specification) {
std::string orderStr = spec.get("triplesOrder");
order= parseOrder(orderStr.c_str());
if(order==Unknown)
order = SPO;
arrayY = IntSequence::getArray(spec.get("stream.y"));
arrayZ = IntSequence::getArray(spec.get("stream.z"));
arrayIndex = NULL;
bitmapY = NULL;
bitmapZ = NULL;
bitmapIndex = NULL;
waveletY = NULL;
predicateCount = NULL;
}
PlainTriples::PlainTriples(HDTSpecification &specification) : spec(specification) {
std::string orderStr = "";
try{
orderStr= spec.get("triplesOrder");
}catch(exception& e){}
order = parseOrder(orderStr.c_str());
if(order==Unknown) {
order = SPO;
}
string typex="";
string typey="";
string typez="";
try{
typex = spec.get("stream.x");
typey = spec.get("stream.y");
typez = spec.get("stream.z");
}catch (std::exception& e){}
streamX = IntSequence::getArray(typex);
streamY = IntSequence::getArray(typey);
streamZ = IntSequence::getArray(typez);
}
TriplesKyoto::TriplesKyoto(HDTSpecification &specification) : spec(specification) {
unlink("triples.kct");
string ord = "";
try{
ord = spec.get("triplesOrder");
}catch(exception& e){}
order = parseOrder(ord.c_str());
if(order==Unknown){
order = SPO;
}
comparator.setOrder(order);
db.tune_comparator(this);
db.tune_options(TreeDB::TLINEAR /*| TreeDB::TCCOMPESS*/);
db.tune_alignment(0);
db.tune_buckets(1LL*1000*1000);
db.tune_map(256LL*1024*1024);
db.tune_page_cache(64LL*1024*1024);
if (!db.open("triples.kct", TreeDB::OWRITER | TreeDB::OCREATE)) {
cerr << "Triples DB open error: " << db.error().name() << endl;
}
}
void BasicHDT::loadTriplesFromHDTs(const char** fileNames, size_t numFiles, const char* baseUri, ProgressListener* listener) {
// Generate Triples
ModifiableTriples* triplesList = new TriplesList(spec);
//ModifiableTriples *triplesList = new TriplesKyoto(spec);
//ModifiableTriples *triplesList = new TripleListDisk();
StopWatch st;
IntermediateListener iListener(listener);
try {
NOTIFY(listener, "Loading Triples", 0, 100);
iListener.setRange(0, 60);
triplesList->startProcessing(&iListener);
TriplesLoader tripLoader(dictionary, triplesList, &iListener);
// FIXME: Import from files
uint64_t totalOriginalSize=0;
BasicHDT hdt;
for(size_t i=0;i<numFiles;i++) {
const char *fileName = fileNames[i];
cout << endl << "Load triples from " << fileName << endl;
hdt.mapHDT(fileName);
Dictionary *dict = hdt.getDictionary();
// Create mapping arrays
cout << "Generating mapping subjects" << endl;
unsigned int nsubjects = dict->getNsubjects();
LogSequence2 subjectMap(bits(dictionary->getNsubjects()), nsubjects);
subjectMap.resize(nsubjects);
for(unsigned int i=0;i<nsubjects;i++) {
string str = dict->idToString(i+1, SUBJECT);
unsigned int newid = dictionary->stringToId(str, SUBJECT);
subjectMap.set(i, newid);
}
cout << "Generating mapping predicates" << endl;
unsigned int npredicates = dict->getNpredicates();
LogSequence2 predicateMap(bits(dictionary->getNpredicates()), npredicates);
predicateMap.resize(npredicates);
for(unsigned int i=0;i<npredicates;i++) {
string str = dict->idToString(i+1, PREDICATE);
unsigned int newid = dictionary->stringToId(str, PREDICATE);
predicateMap.set(i, newid);
}
cout << "Generating mapping objects" << endl;
unsigned int nobjects = dict->getNobjects();
LogSequence2 objectMap(bits(dictionary->getNobjects()), nobjects);
objectMap.resize(nobjects);
for(unsigned int i=0;i<nobjects;i++) {
string str = dict->idToString(i+1, OBJECT);
unsigned int newid = dictionary->stringToId(str, OBJECT);
objectMap.set(i, newid);
}
totalOriginalSize += hdt.getHeader()->getPropertyLong("_:statistics", HDTVocabulary::ORIGINAL_SIZE.c_str());
size_t numtriples = hdt.getTriples()->getNumberOfElements();
IteratorTripleID *it = hdt.getTriples()->searchAll();
TripleID newTid;
char str[100];
long long int j = 0;
while(it->hasNext()) {
TripleID *tid = it->next();
newTid.setAll(
(unsigned int)subjectMap.get(tid->getSubject()-1),
(unsigned int)predicateMap.get(tid->getPredicate()-1),
(unsigned int)objectMap.get(tid->getObject()-1)
);
triplesList->insert(newTid);
if ((listener != NULL) && (j % 100000) == 0) {
sprintf(str, "%lld triples added.", j);
listener->notifyProgress((j*100)/numtriples, str);
}
j++;
}
delete it;
}
triplesList->stopProcessing(&iListener);
// SORT & Duplicates
TripleComponentOrder order = parseOrder(spec.get("triplesOrder").c_str());
if (order == Unknown) {
order = SPO;
}
iListener.setRange(80, 85);
triplesList->sort(order, &iListener);
iListener.setRange(85, 90);
triplesList->removeDuplicates(&iListener);
header->insert("_:statistics", HDTVocabulary::ORIGINAL_SIZE, totalOriginalSize);
} catch (const char *e) {
cout << "Catch exception triples" << e << endl;
delete triplesList;
throw e;
} catch (char *e) {
cout << "Catch exception triples" << e << endl;
delete triplesList;
throw e;
}
if (triples->getType() == triplesList->getType()) {
delete triples;
triples = triplesList;
} else {
iListener.setRange(90, 100);
try {
triples->load(*triplesList, &iListener);
} catch (const char* e) {
delete triplesList;
throw e;
}
delete triplesList;
}
//cout << triples->getNumberOfElements() << " triples added in " << st << endl << endl;
}
int
parseResReq(char *resReq,
struct resVal *resVal,
struct lsInfo *lsInfo,
int options)
{
static char fname[] = "parseResReq()";
int cc;
struct sections reqSect;
if (logclass & (LC_TRACE | LC_SCHED))
ls_syslog(LOG_DEBUG3, "%s: resReq=%s", fname, resReq);
initResVal(resVal);
ALLOC_STRING(resVal->selectStr,
resVal->selectStrSize,
MAX(3*strlen(resReq) + 1, MAXLINELEN + MAXLSFNAMELEN));
if ((cc = parseSection(resReq, &reqSect)) != PARSE_OK)
return cc;
if ((cc = setDefaults(resVal, lsInfo, options)) < 0)
return cc;
if (options & PR_SELECT) {
if (options & PR_XOR) {
if ((cc = parseSelect(reqSect.select,
resVal,
lsInfo,
TRUE,
options)) != PARSE_OK)
return cc;
} else {
if ((cc = parseSelect(reqSect.select,
resVal,
lsInfo,
FALSE,
options)) != PARSE_OK)
return cc;
}
}
if (options & PR_ORDER) {
if ((cc = parseOrder(reqSect.order,
resVal,
lsInfo)) != PARSE_OK)
return cc;
}
if (options & PR_RUSAGE) {
if ((cc = parseUsage(reqSect.rusage,
resVal,
lsInfo)) != PARSE_OK)
return cc;
}
if (options & PR_FILTER) {
if ((cc = parseFilter(reqSect.filter,
resVal,
lsInfo)) != PARSE_OK)
return cc;
}
if (options & PR_SPAN) {
if ((cc = parseSpan(reqSect.span,
resVal)) != PARSE_OK)
return cc;
}
return(PARSE_OK);
}