void ParserVocabulary::addQualifiedName(QualifiedNameTable* table, const QualifiedName& name)
{
table->addUnique(name); // Should be last action?
if(!name._prefix._stringIndex &&
name._prefix._literalCharacterString.length())
{
addStringToTable(PREFIX, name._prefix._literalCharacterString);
}
if(!name._namespaceName._stringIndex &&
name._namespaceName._literalCharacterString.length())
{
addStringToTable(NAMESPACE_NAME, name._namespaceName._literalCharacterString);
}
if(!name._localName._stringIndex)
{
addStringToTable(LOCAL_NAME, name._localName._literalCharacterString);
}
}
void OsmPbfWriterPrivate::addEntityStringsToTable(QHash<QString, int32_t> *stringTableHash, const OsmStructures::EntityBase &entity, bool sortHashes)
{
if (sortHashes) {
QStringList sortedKeys = entity.tags.keys();
qSort(sortedKeys);
for (QStringList::const_iterator iTagKeyString = sortedKeys.constBegin() ; iTagKeyString != sortedKeys.constEnd() ; ++iTagKeyString) {
addStringToTable(stringTableHash, *iTagKeyString);
addStringToTable(stringTableHash, entity.tags.value(*iTagKeyString));
}
} else {
for (QHash<QString, QString>::const_iterator iTagString = entity.tags.constBegin() ; iTagString != entity.tags.constEnd() ; ++iTagString) {
addStringToTable(stringTableHash, iTagString.key());
addStringToTable(stringTableHash, iTagString.value());
}
}
if (!entity.user.isNull())
addStringToTable(stringTableHash, entity.user);
}
void OsmPbfWriterPrivate::writeBlock(bool forceWrite)
{
if ( (forceWrite && (nodesList.count() + waysList.count() + relationsList.count() > 0) ) ||
(nodesList.count() + waysList.count() + relationsList.count() >= entitiesPerBlockLimit) ) {
OSMPBF::PrimitiveBlock currentFileBlockParsed;
int32_t granularityCoordinate = 100, granularityDate = 1000;
int64_t offsetLat = 0, offsetLon = 0;
//FIXME: keep order of addition
//Extract all strings to hash for future reference
//FIXME extract granularities and offset
QHash<QString, int32_t> stringTableHash;
for (QList<OsmStructures::Node>::const_iterator iNode = nodesList.constBegin(); iNode != nodesList.constEnd() ; ++iNode)
addEntityStringsToTable(&stringTableHash, *iNode);
for (QList<OsmStructures::Way>::const_iterator iWay = waysList.constBegin(); iWay != waysList.constEnd() ; ++iWay)
addEntityStringsToTable(&stringTableHash, *iWay);
for (QList<OsmStructures::Relation>::const_iterator iRelation = relationsList.constBegin(); iRelation != relationsList.constEnd() ; ++iRelation) {
addEntityStringsToTable(&stringTableHash, *iRelation);
for (QSet<OsmStructures::RelationMember>::const_iterator iRelationMember = iRelation->members.constBegin(); iRelationMember != iRelation->members.constEnd() ; ++iRelationMember) {
addStringToTable(&stringTableHash, iRelationMember->role);
}
}
//Create list of strings from hash (sorted by value)
int32_t stringsCount = stringTableHash.count() + 1;
QScopedArrayPointer<QString> stringTableList(new QString[stringsCount]);
stringTableList[0] = QString("");
for (QHash<QString, int32_t>::const_iterator iString = stringTableHash.constBegin() ; iString != stringTableHash.constEnd() ; ++iString) {
stringTableList[iString.value()] = iString.key();
}
//Write list of string to block stringtable
for (int iIndex = 0 ; iIndex < stringsCount ; iIndex++) {
currentFileBlockParsed.mutable_stringtable()->add_s(stringTableList[iIndex].toUtf8().data(), stringTableList[iIndex].toUtf8().length());
}
stringTableList.reset();
//Write granularities and offsets
if (granularityCoordinate != 100)
currentFileBlockParsed.set_granularity(granularityCoordinate);
if (granularityDate != 1000)
currentFileBlockParsed.set_date_granularity(granularityDate);
if (offsetLat != 0)
currentFileBlockParsed.set_lat_offset(offsetLat);
if (offsetLon != 0)
currentFileBlockParsed.set_lon_offset(offsetLon);
//Write nodes
if (nodesList.count() > 0) {
OSMPBF::PrimitiveGroup* nodesGroup = currentFileBlockParsed.add_primitivegroup();
int64_t denseId = 0, denseLat = 0, denseLon = 0;
int64_t denseTimestamp = 0, denseChangeset = 0, denseUid = 0, denseUser = 0;
for (QList<OsmStructures::Node>::const_iterator iNode = nodesList.constBegin(); iNode != nodesList.constEnd() ; ++iNode) {
if ( ( (iNode->getLatNanoDegree() - offsetLat) % granularityCoordinate ) ||
(iNode->getLonNanoDegree() - offsetLon) % granularityCoordinate )
qFatal("Loosing precision");
int64_t lat_int = (iNode->getLatNanoDegree() - offsetLat) / granularityCoordinate;
int64_t lon_int = (iNode->getLonNanoDegree() - offsetLon) / granularityCoordinate;
if (useDenseNodes) {
OSMPBF::DenseNodes* pbfDense = nodesGroup->mutable_dense();
pbfDense->add_id(iNode->id - denseId);
pbfDense->add_lat(lat_int - denseLat);
pbfDense->add_lon(lon_int - denseLon);
denseId = iNode->id;
denseLat = lat_int;
denseLon = lon_int;
pbfDense->mutable_denseinfo()->add_version(iNode->version);
if (iNode->timestamp.toMSecsSinceEpoch() % granularityDate)
qFatal("Loosing precision");
int64_t timestamp_value = iNode->timestamp.toMSecsSinceEpoch() / granularityDate;
int32_t user_sid = stringTableHash.value(iNode->user);
pbfDense->mutable_denseinfo()->add_timestamp(timestamp_value - denseTimestamp);
pbfDense->mutable_denseinfo()->add_changeset(iNode->changeset - denseChangeset);
pbfDense->mutable_denseinfo()->add_uid(iNode->uid - denseUid);
pbfDense->mutable_denseinfo()->add_user_sid(user_sid - denseUser);
denseTimestamp = timestamp_value;
denseChangeset = iNode->changeset;
denseUid = iNode->uid;
denseUser = user_sid;
for (QHash<QString, QString>::const_iterator iTagString = iNode->tags.constBegin() ; iTagString != iNode->tags.constEnd() ; ++iTagString) {
pbfDense->add_keys_vals(stringTableHash.value(iTagString.key()));
pbfDense->add_keys_vals(stringTableHash.value(iTagString.value()));
}
pbfDense->add_keys_vals(0);
} else {
OSMPBF::Node* pbfNode = nodesGroup->add_nodes();
pbfNode->set_id(iNode->id);
pbfNode->set_lon(lat_int);
pbfNode->set_lat(lon_int);
packEntityInfo(pbfNode->mutable_info(), *iNode, granularityDate, stringTableHash);
for (QHash<QString, QString>::const_iterator iTagString = iNode->tags.constBegin() ; iTagString != iNode->tags.constEnd() ; ++iTagString) {
pbfNode->add_keys(stringTableHash.value(iTagString.key()));
pbfNode->add_vals(stringTableHash.value(iTagString.value()));
}
}
}
}
//write ways
if (waysList.count() > 0) {
OSMPBF::PrimitiveGroup* waysGroup = currentFileBlockParsed.add_primitivegroup();
for (QList<OsmStructures::Way>::const_iterator iWay = waysList.constBegin(); iWay != waysList.constEnd() ; ++iWay) {
OSMPBF::Way* pbfWay = waysGroup->add_ways();
pbfWay->set_id(iWay->id);
int64_t currentNodeId = 0;
for (QList<int64_t>::const_iterator iNodeId = iWay->nodes.constBegin() ; iNodeId != iWay->nodes.constEnd() ; ++iNodeId) {
pbfWay->add_refs(*iNodeId - currentNodeId);
currentNodeId = *iNodeId;
}
packEntityInfo(pbfWay->mutable_info(), *iWay, granularityDate, stringTableHash);
for (QHash<QString, QString>::const_iterator iTagString = iWay->tags.constBegin() ; iTagString != iWay->tags.constEnd() ; ++iTagString) {
pbfWay->add_keys(stringTableHash.value(iTagString.key()));
pbfWay->add_vals(stringTableHash.value(iTagString.value()));
}
}
}
//write relations
if (relationsList.count() > 0) {
OSMPBF::PrimitiveGroup* relationsGroup = currentFileBlockParsed.add_primitivegroup();
for (QList<OsmStructures::Relation>::const_iterator iRelation = relationsList.constBegin(); iRelation != relationsList.constEnd() ; ++iRelation) {
OSMPBF::Relation *pbfRelation = relationsGroup->add_relations();
pbfRelation->set_id(iRelation->id);
int64_t currentMemberId = 0;
for (QSet<OsmStructures::RelationMember>::const_iterator iRelationMember = iRelation->members.constBegin(); iRelationMember != iRelation->members.constEnd() ; ++iRelationMember) {
pbfRelation->add_roles_sid(stringTableHash.value(iRelationMember->role));
pbfRelation->add_memids(iRelationMember->memberRef.id - currentMemberId);
currentMemberId = iRelationMember->memberRef.id;
OSMPBF::Relation_MemberType currentType;
switch (iRelationMember->memberRef.type) {
case OsmStructures::EnumOsmNode: currentType = OSMPBF::Relation_MemberType_NODE; break;
case OsmStructures::EnumOsmWay: currentType = OSMPBF::Relation_MemberType_WAY; break;
case OsmStructures::EnumOsmRelation: currentType = OSMPBF::Relation_MemberType_RELATION; break;
default:
throw std::exception();
break;
}
pbfRelation->add_types(currentType);
}
packEntityInfo(pbfRelation->mutable_info(), *iRelation, granularityDate, stringTableHash);
for (QHash<QString, QString>::const_iterator iTagString = iRelation->tags.constBegin() ; iTagString != iRelation->tags.constEnd() ; ++iTagString) {
pbfRelation->add_keys(stringTableHash.value(iTagString.key()));
pbfRelation->add_vals(stringTableHash.value(iTagString.value()));
}
}
}
QByteArray currentFileBlockPacked = packPbfMessageToArray(currentFileBlockParsed);
writeFileblock("OSMData", currentFileBlockPacked);
nodesList.clear();
waysList.clear();
relationsList.clear();
}
}