コード例 #1
0
ファイル: BasicHDT.cpp プロジェクト: mavlyutovrus/uriencoding
void BasicHDT::loadTriples(const char* fileName, const char* baseUri, RDFNotation notation, ProgressListener* listener) {
	// Generate Triples
	ModifiableTriples* triplesList = new TriplesList(spec);
	//ModifiableTriples *triplesList = new TriplesKyoto(spec);
	//ModifiableTriples *triplesList = new TripleListDisk();
	StopWatch st;
	IntermediateListener iListener(listener);
	try {
		NOTIFY(listener, "Loading Triples", 0, 100);
		iListener.setRange(0, 60);

		triplesList->startProcessing(&iListener);

		TriplesLoader tripLoader(dictionary, triplesList, &iListener);

		RDFParserCallback *pars = RDFParserCallback::getParserCallback(
				notation);
		pars->doParse(fileName, baseUri, notation, &tripLoader);
		delete pars;
		header->insert("_:statistics", HDTVocabulary::ORIGINAL_SIZE, tripLoader.getSize());

		triplesList->stopProcessing(&iListener);

		// SORT & Duplicates
		TripleComponentOrder order = parseOrder(
				spec.get("triplesOrder").c_str());
		if (order == Unknown) {
			order = SPO;
		}

		iListener.setRange(80, 85);
		triplesList->sort(order, &iListener);

		iListener.setRange(85, 90);
		triplesList->removeDuplicates(&iListener);
	} catch (const char *e) {
		cout << "Catch exception triples" << e << endl;
		delete triplesList;
		throw e;
	} catch (char *e) {
		cout << "Catch exception triples" << e << endl;
		delete triplesList;
		throw e;
	}
	if (triples->getType() == triplesList->getType()) {
		delete triples;
		triples = triplesList;
	} else {
		iListener.setRange(90, 100);
		try {
			triples->load(*triplesList, &iListener);
		} catch (const char* e) {
			delete triplesList;
			throw e;
		}
		delete triplesList;
	}

	//cout << triples->getNumberOfElements() << " triples added in " << st << endl << endl;
}
コード例 #2
0
CompactTriples::CompactTriples(HDTSpecification &specification) : numTriples(0), spec(specification) {
	std::string orderStr = spec.get("triplesOrder");
	order= parseOrder(orderStr.c_str());
	if(order==Unknown)
		order = SPO;

	streamY = IntSequence::getArray(spec.get("stream.y"));
	streamZ = IntSequence::getArray(spec.get("stream.z"));
}
コード例 #3
0
BitmapTriples::BitmapTriples(HDTSpecification &specification) : spec(specification) {
	std::string orderStr = spec.get("triplesOrder");
	order= parseOrder(orderStr.c_str());
	if(order==Unknown)
		order = SPO;
	arrayY = IntSequence::getArray(spec.get("stream.y"));
	arrayZ = IntSequence::getArray(spec.get("stream.z"));
	arrayIndex = NULL;
	bitmapY = NULL;
	bitmapZ = NULL;
	bitmapIndex = NULL;
	waveletY = NULL;
	predicateCount = NULL;
}
コード例 #4
0
ファイル: PlainTriples.cpp プロジェクト: rdmpage/hdt-cpp
PlainTriples::PlainTriples(HDTSpecification &specification) : spec(specification) {
	std::string orderStr = "";
	try{
		orderStr= spec.get("triplesOrder");
	}catch(exception& e){}
	order = parseOrder(orderStr.c_str());
	if(order==Unknown) {
			order = SPO;
	}
	string typex="";
	string typey="";
	string typez="";
	try{
		typex = spec.get("stream.x");
		typey = spec.get("stream.y");
		typez = spec.get("stream.z");
	}catch (std::exception& e){}
	streamX = IntSequence::getArray(typex);
	streamY = IntSequence::getArray(typey);
	streamZ = IntSequence::getArray(typez);
}
コード例 #5
0
ファイル: TriplesKyoto.cpp プロジェクト: rdmpage/hdt-cpp
TriplesKyoto::TriplesKyoto(HDTSpecification &specification) : spec(specification) {
	unlink("triples.kct");
	string ord = "";
	try{
		ord = spec.get("triplesOrder");
	}catch(exception& e){}
    order = parseOrder(ord.c_str());
    if(order==Unknown){
        order = SPO;
    }

    comparator.setOrder(order);

    db.tune_comparator(this);
	db.tune_options(TreeDB::TLINEAR /*| TreeDB::TCCOMPESS*/);
	db.tune_alignment(0);
    db.tune_buckets(1LL*1000*1000);
    db.tune_map(256LL*1024*1024);
    db.tune_page_cache(64LL*1024*1024);
	if (!db.open("triples.kct", TreeDB::OWRITER | TreeDB::OCREATE)) {
		cerr << "Triples DB open error: " << db.error().name() << endl;
	}
}
コード例 #6
0
ファイル: BasicHDT.cpp プロジェクト: akjoshi/hdt-it
void BasicHDT::loadTriplesFromHDTs(const char** fileNames, size_t numFiles, const char* baseUri, ProgressListener* listener) {
	// Generate Triples
	ModifiableTriples* triplesList = new TriplesList(spec);
	//ModifiableTriples *triplesList = new TriplesKyoto(spec);
	//ModifiableTriples *triplesList = new TripleListDisk();
	StopWatch st;
	IntermediateListener iListener(listener);
	try {
		NOTIFY(listener, "Loading Triples", 0, 100);
		iListener.setRange(0, 60);

		triplesList->startProcessing(&iListener);

		TriplesLoader tripLoader(dictionary, triplesList, &iListener);

		// FIXME: Import from files

		uint64_t totalOriginalSize=0;
		BasicHDT hdt;

		for(size_t i=0;i<numFiles;i++) {
			const char *fileName = fileNames[i];
	        cout << endl << "Load triples from " << fileName << endl;
	        hdt.mapHDT(fileName);
	        Dictionary *dict = hdt.getDictionary();

	        // Create mapping arrays
	        cout << "Generating mapping subjects" << endl;
	        unsigned int nsubjects = dict->getNsubjects();
	        LogSequence2 subjectMap(bits(dictionary->getNsubjects()), nsubjects);
	        subjectMap.resize(nsubjects);
	        for(unsigned int i=0;i<nsubjects;i++) {
	        	string str = dict->idToString(i+1, SUBJECT);
	        	unsigned int newid = dictionary->stringToId(str, SUBJECT);
	        	subjectMap.set(i, newid);
	        }

	        cout << "Generating mapping predicates" << endl;
	        unsigned int npredicates = dict->getNpredicates();
	        LogSequence2 predicateMap(bits(dictionary->getNpredicates()), npredicates);
	        predicateMap.resize(npredicates);
	        for(unsigned int i=0;i<npredicates;i++) {
	        	string str = dict->idToString(i+1, PREDICATE);
	        	unsigned int newid = dictionary->stringToId(str, PREDICATE);
	        	predicateMap.set(i, newid);
	        }

	        cout << "Generating mapping objects" << endl;
	        unsigned int nobjects = dict->getNobjects();
	        LogSequence2 objectMap(bits(dictionary->getNobjects()), nobjects);
	        objectMap.resize(nobjects);
	        for(unsigned int i=0;i<nobjects;i++) {
	        	string str = dict->idToString(i+1, OBJECT);
	        	unsigned int newid = dictionary->stringToId(str, OBJECT);
	        	objectMap.set(i, newid);
	        }

	        totalOriginalSize += hdt.getHeader()->getPropertyLong("_:statistics", HDTVocabulary::ORIGINAL_SIZE.c_str());

	        size_t numtriples = hdt.getTriples()->getNumberOfElements();
	        IteratorTripleID *it = hdt.getTriples()->searchAll();

	        TripleID newTid;
	        char str[100];
	        long long int j = 0;
	        while(it->hasNext()) {
	        	TripleID *tid = it->next();

	        	newTid.setAll(
	        			(unsigned int)subjectMap.get(tid->getSubject()-1),
	        			(unsigned int)predicateMap.get(tid->getPredicate()-1),
	        			(unsigned int)objectMap.get(tid->getObject()-1)
	        			);

	        	triplesList->insert(newTid);

	        	if ((listener != NULL) && (j % 100000) == 0) {
	        		sprintf(str, "%lld triples added.", j);
	        		listener->notifyProgress((j*100)/numtriples, str);
	        	}
	            j++;
	        }
	        delete it;
		}

		triplesList->stopProcessing(&iListener);

		// SORT & Duplicates
		TripleComponentOrder order = parseOrder(spec.get("triplesOrder").c_str());
		if (order == Unknown) {
			order = SPO;
		}

		iListener.setRange(80, 85);
		triplesList->sort(order, &iListener);

		iListener.setRange(85, 90);
		triplesList->removeDuplicates(&iListener);

		header->insert("_:statistics", HDTVocabulary::ORIGINAL_SIZE, totalOriginalSize);
	} catch (const char *e) {
		cout << "Catch exception triples" << e << endl;
		delete triplesList;
		throw e;
	} catch (char *e) {
		cout << "Catch exception triples" << e << endl;
		delete triplesList;
		throw e;
	}
	if (triples->getType() == triplesList->getType()) {
		delete triples;
		triples = triplesList;
	} else {
		iListener.setRange(90, 100);
		try {
			triples->load(*triplesList, &iListener);
		} catch (const char* e) {
			delete triplesList;
			throw e;
		}
		delete triplesList;
	}

	//cout << triples->getNumberOfElements() << " triples added in " << st << endl << endl;

}
コード例 #7
0
ファイル: resreq.c プロジェクト: CraigNoble/openlava
int
parseResReq(char *resReq,
            struct resVal *resVal,
            struct lsInfo *lsInfo,
            int options)
{
    static char       fname[] = "parseResReq()";
    int               cc;
    struct sections   reqSect;

    if (logclass & (LC_TRACE | LC_SCHED))
        ls_syslog(LOG_DEBUG3, "%s: resReq=%s", fname, resReq);

    initResVal(resVal);

    ALLOC_STRING(resVal->selectStr,
                 resVal->selectStrSize,
                 MAX(3*strlen(resReq) + 1, MAXLINELEN + MAXLSFNAMELEN));

    if ((cc = parseSection(resReq, &reqSect)) != PARSE_OK)
        return cc;

    if ((cc = setDefaults(resVal, lsInfo, options)) < 0)
        return cc;

    if (options & PR_SELECT) {

        if (options & PR_XOR) {
            if ((cc = parseSelect(reqSect.select,
                                  resVal,
                                  lsInfo,
                                  TRUE,
                                  options)) != PARSE_OK)
                return cc;
        } else {
            if ((cc = parseSelect(reqSect.select,
                                  resVal,
                                  lsInfo,
                                  FALSE,
                                  options)) != PARSE_OK)
                return cc;
	}
    }

    if (options & PR_ORDER) {
        if ((cc = parseOrder(reqSect.order,
                             resVal,
                             lsInfo)) != PARSE_OK)
            return cc;
    }

    if (options & PR_RUSAGE) {
        if ((cc = parseUsage(reqSect.rusage,
                             resVal,
                             lsInfo)) != PARSE_OK)
            return cc;
    }

    if (options & PR_FILTER) {
        if ((cc = parseFilter(reqSect.filter,
                              resVal,
                              lsInfo)) != PARSE_OK)
            return cc;
    }

    if (options & PR_SPAN) {
        if ((cc = parseSpan(reqSect.span,
                            resVal)) != PARSE_OK)
            return cc;
    }

    return(PARSE_OK);
}