void OsmAnd::ObfMapSectionReader_P::readTreeNodeChildren( const std::unique_ptr<ObfReader_P>& reader, const std::shared_ptr<const ObfMapSectionInfo>& section, const std::shared_ptr<ObfMapSectionLevelTreeNode>& treeNode, MapFoundationType& foundation, QList< std::shared_ptr<ObfMapSectionLevelTreeNode> >* nodesWithData, const AreaI* bbox31, IQueryController* controller) { auto cis = reader->_codedInputStream.get(); for(;;) { auto tag = cis->ReadTag(); switch(gpb::internal::WireFormatLite::GetTagFieldNumber(tag)) { case 0: return; case OBF::OsmAndMapIndex_MapDataBox::kBoxesFieldNumber: { auto length = ObfReaderUtilities::readBigEndianInt(cis); auto offset = cis->CurrentPosition(); auto oldLimit = cis->PushLimit(length); std::shared_ptr<ObfMapSectionLevelTreeNode> childNode(new ObfMapSectionLevelTreeNode()); childNode->_foundation = treeNode->_foundation; childNode->_offset = offset; childNode->_length = length; readTreeNode(reader, section, treeNode->_area31, childNode); if(bbox31 && !bbox31->intersects(childNode->_area31)) { cis->Skip(cis->BytesUntilLimit()); cis->PopLimit(oldLimit); break; } cis->PopLimit(oldLimit); if(childNode->_foundation != MapFoundationType::Undefined) { if(foundation == MapFoundationType::Undefined) foundation = childNode->_foundation; else if(foundation != childNode->_foundation) foundation = MapFoundationType::Mixed; } if(nodesWithData && childNode->_dataOffset > 0) nodesWithData->push_back(childNode); cis->Seek(offset); oldLimit = cis->PushLimit(length); cis->Skip(childNode->_childrenInnerOffset); readTreeNodeChildren(reader, section, childNode, foundation, nodesWithData, bbox31, controller); assert(cis->BytesUntilLimit() == 0); cis->PopLimit(oldLimit); } break; default: ObfReaderUtilities::skipUnknownField(cis, tag); break; } } }
void OsmAnd::ObfMapSectionReader_P::loadMapObjects( const ObfReader_P& reader, const std::shared_ptr<const ObfMapSectionInfo>& section, ZoomLevel zoom, const AreaI* bbox31, QList< std::shared_ptr<const OsmAnd::BinaryMapObject> >* resultOut, MapSurfaceType* outBBoxOrSectionSurfaceType, const ObfMapSectionReader::FilterByIdFunction filterById, const VisitorFunction visitor, DataBlocksCache* cache, QList< std::shared_ptr<const DataBlock> >* outReferencedCacheEntries, const std::shared_ptr<const IQueryController>& queryController, ObfMapSectionReader_Metrics::Metric_loadMapObjects* const metric) { const auto cis = reader.getCodedInputStream().get(); const auto filterReadById = [filterById, zoom] (const std::shared_ptr<const ObfMapSectionInfo>& section, const ObfObjectId mapObjectId, const AreaI& bbox, const ZoomLevel firstZoomLevel, const ZoomLevel lastZoomLevel) -> bool { return filterById(section, mapObjectId, bbox, firstZoomLevel, lastZoomLevel, zoom); }; // Ensure encoding/decoding rules are read if (section->_p->_attributeMappingLoaded.loadAcquire() == 0) { QMutexLocker scopedLocker(§ion->_p->_attributeMappingLoadMutex); if (!section->_p->_attributeMapping) { // Read encoding/decoding rules cis->Seek(section->offset); auto oldLimit = cis->PushLimit(section->length); const std::shared_ptr<ObfMapSectionAttributeMapping> attributeMapping(new ObfMapSectionAttributeMapping()); readAttributeMapping(reader, section, attributeMapping); section->_p->_attributeMapping = attributeMapping; cis->PopLimit(oldLimit); section->_p->_attributeMappingLoaded.storeRelease(1); } } ObfMapSectionReader_Metrics::Metric_loadMapObjects localMetric; auto bboxOrSectionSurfaceType = MapSurfaceType::Undefined; if (outBBoxOrSectionSurfaceType) *outBBoxOrSectionSurfaceType = bboxOrSectionSurfaceType; QList< std::shared_ptr<const DataBlock> > danglingReferencedCacheEntries; for (const auto& mapLevel : constOf(section->levels)) { // Update metric if (metric) metric->visitedLevels++; if (mapLevel->minZoom > zoom || mapLevel->maxZoom < zoom) continue; if (bbox31) { const Stopwatch bboxLevelCheckStopwatch(metric != nullptr); const auto shouldSkip = !bbox31->contains(mapLevel->area31) && !mapLevel->area31.contains(*bbox31) && !bbox31->intersects(mapLevel->area31); if (metric) metric->elapsedTimeForLevelsBbox += bboxLevelCheckStopwatch.elapsed(); if (shouldSkip) continue; } const Stopwatch treeNodesStopwatch(metric != nullptr); if (metric) metric->acceptedLevels++; // If there are no tree nodes in map level, it means they are not loaded. // Since loading may be called from multiple threads, loading of root nodes needs synchronization // Ensure encoding/decoding rules are read if (mapLevel->_p->_rootNodesLoaded.loadAcquire() == 0) { QMutexLocker scopedLocker(&mapLevel->_p->_rootNodesLoadMutex); if (!mapLevel->_p->_rootNodes) { cis->Seek(mapLevel->offset); auto oldLimit = cis->PushLimit(mapLevel->length); cis->Skip(mapLevel->firstDataBoxInnerOffset); const std::shared_ptr< QList< std::shared_ptr<const ObfMapSectionLevelTreeNode> > > rootNodes( new QList< std::shared_ptr<const ObfMapSectionLevelTreeNode> >()); readMapLevelTreeNodes(reader, section, mapLevel, *rootNodes); mapLevel->_p->_rootNodes = rootNodes; cis->PopLimit(oldLimit); mapLevel->_p->_rootNodesLoaded.storeRelease(1); } } // Collect tree nodes with data QList< std::shared_ptr<const ObfMapSectionLevelTreeNode> > treeNodesWithData; for (const auto& rootNode : constOf(*mapLevel->_p->_rootNodes)) { // Update metric if (metric) metric->visitedNodes++; if (bbox31) { const Stopwatch bboxNodeCheckStopwatch(metric != nullptr); const auto shouldSkip = !bbox31->contains(rootNode->area31) && !rootNode->area31.contains(*bbox31) && !bbox31->intersects(rootNode->area31); // Update metric if (metric) metric->elapsedTimeForNodesBbox += bboxNodeCheckStopwatch.elapsed(); if (shouldSkip) continue; } // Update metric if (metric) metric->acceptedNodes++; if (rootNode->dataOffset > 0) treeNodesWithData.push_back(rootNode); auto rootSubnodesSurfaceType = MapSurfaceType::Undefined; if (rootNode->hasChildrenDataBoxes) { cis->Seek(rootNode->offset); auto oldLimit = cis->PushLimit(rootNode->length); cis->Skip(rootNode->firstDataBoxInnerOffset); readTreeNodeChildren(reader, section, rootNode, rootSubnodesSurfaceType, &treeNodesWithData, bbox31, queryController, metric); ObfReaderUtilities::ensureAllDataWasRead(cis); cis->PopLimit(oldLimit); } const auto surfaceTypeToMerge = (rootSubnodesSurfaceType != MapSurfaceType::Undefined) ? rootSubnodesSurfaceType : rootNode->surfaceType; if (surfaceTypeToMerge != MapSurfaceType::Undefined) { if (bboxOrSectionSurfaceType == MapSurfaceType::Undefined) bboxOrSectionSurfaceType = surfaceTypeToMerge; else if (bboxOrSectionSurfaceType != surfaceTypeToMerge) bboxOrSectionSurfaceType = MapSurfaceType::Mixed; } } // Sort blocks by data offset to force forward-only seeking std::sort(treeNodesWithData, [] (const std::shared_ptr<const ObfMapSectionLevelTreeNode>& l, const std::shared_ptr<const ObfMapSectionLevelTreeNode>& r) -> bool { return l->dataOffset < r->dataOffset; }); // Update metric const Stopwatch mapObjectsStopwatch(metric != nullptr); if (metric) metric->elapsedTimeForNodes += treeNodesStopwatch.elapsed(); // Read map objects from their blocks for (const auto& treeNode : constOf(treeNodesWithData)) { if (queryController && queryController->isAborted()) break; DataBlockId blockId; blockId.sectionRuntimeGeneratedId = section->runtimeGeneratedId; blockId.offset = treeNode->dataOffset; if (cache && cache->shouldCacheBlock(blockId, treeNode->area31, bbox31)) { // In case cache is provided, read and cache const auto levelZooms = Utilities::enumerateZoomLevels(treeNode->level->minZoom, treeNode->level->maxZoom); std::shared_ptr<const DataBlock> dataBlock; std::shared_ptr<const DataBlock> sharedBlockReference; proper::shared_future< std::shared_ptr<const DataBlock> > futureSharedBlockReference; if (cache->obtainReferenceOrFutureReferenceOrMakePromise(blockId, zoom, levelZooms, sharedBlockReference, futureSharedBlockReference)) { // Got reference or future reference // Update metric if (metric) metric->mapObjectsBlocksReferenced++; if (sharedBlockReference) { // Ok, this block was already loaded, just use it dataBlock = sharedBlockReference; } else { // Wait until it will be loaded dataBlock = futureSharedBlockReference.get(); } } else { // Made a promise, so load entire block into temporary storage QList< std::shared_ptr<const BinaryMapObject> > mapObjects; cis->Seek(treeNode->dataOffset); gpb::uint32 length; cis->ReadVarint32(&length); const auto oldLimit = cis->PushLimit(length); readMapObjectsBlock( reader, section, treeNode, &mapObjects, nullptr, nullptr, nullptr, nullptr, metric ? &localMetric : nullptr); ObfReaderUtilities::ensureAllDataWasRead(cis); cis->PopLimit(oldLimit); // Update metric if (metric) metric->mapObjectsBlocksRead++; // Create a data block and share it dataBlock.reset(new DataBlock(blockId, treeNode->area31, treeNode->surfaceType, mapObjects)); cache->fulfilPromiseAndReference(blockId, levelZooms, dataBlock); } if (outReferencedCacheEntries) outReferencedCacheEntries->push_back(dataBlock); else danglingReferencedCacheEntries.push_back(dataBlock); // Process data block for (const auto& mapObject : constOf(dataBlock->mapObjects)) { ////////////////////////////////////////////////////////////////////////// //if (mapObject->id.getOsmId() == 49048972u) //{ // if (bbox31 && *bbox31 == Utilities::tileBoundingBox31(TileId::fromXY(1052, 673), ZoomLevel11)) // { // const auto t = mapObject->toString(); // int i = 5; // } //} ////////////////////////////////////////////////////////////////////////// if (metric) metric->visitedMapObjects++; if (bbox31) { const auto shouldNotSkip = mapObject->bbox31.contains(*bbox31) || bbox31->intersects(mapObject->bbox31); if (!shouldNotSkip) continue; } // Check if map object is desired const auto shouldReject = filterById && !filterById( section, mapObject->id, mapObject->bbox31, mapObject->level->minZoom, mapObject->level->maxZoom, zoom); if (shouldReject) continue; if (!visitor || visitor(mapObject)) { if (metric) metric->acceptedMapObjects++; if (resultOut) resultOut->push_back(qMove(mapObject)); } } } else { // In case there's no cache, simply read cis->Seek(treeNode->dataOffset); gpb::uint32 length; cis->ReadVarint32(&length); const auto oldLimit = cis->PushLimit(length); readMapObjectsBlock( reader, section, treeNode, resultOut, bbox31, filterById != nullptr ? filterReadById : FilterReadingByIdFunction(), visitor, queryController, metric); ObfReaderUtilities::ensureAllDataWasRead(cis); cis->PopLimit(oldLimit); // Update metric if (metric) metric->mapObjectsBlocksRead++; } // Update metric if (metric) metric->mapObjectsBlocksProcessed++; } // Update metric if (metric) metric->elapsedTimeForMapObjectsBlocks += mapObjectsStopwatch.elapsed(); } // In case cache was supplied, but referenced cache entries output collection was not specified, // release all dangling references if (cache && !outReferencedCacheEntries) { for (auto& referencedCacheEntry : danglingReferencedCacheEntries) cache->releaseReference(referencedCacheEntry->id, zoom, referencedCacheEntry); danglingReferencedCacheEntries.clear(); } if (outBBoxOrSectionSurfaceType) *outBBoxOrSectionSurfaceType = bboxOrSectionSurfaceType; // In case cache was used, and metric was requested, some values must be taken from different parts if (cache && metric) { metric->elapsedTimeForOnlyAcceptedMapObjects += localMetric.elapsedTimeForOnlyAcceptedMapObjects; } }
void OsmAnd::ObfMapSectionReader_P::readTreeNodeChildren( const ObfReader_P& reader, const std::shared_ptr<const ObfMapSectionInfo>& section, const std::shared_ptr<const ObfMapSectionLevelTreeNode>& treeNode, MapSurfaceType& outChildrenSurfaceType, QList< std::shared_ptr<const ObfMapSectionLevelTreeNode> >* nodesWithData, const AreaI* bbox31, const std::shared_ptr<const IQueryController>& queryController, ObfMapSectionReader_Metrics::Metric_loadMapObjects* const metric) { const auto cis = reader.getCodedInputStream().get(); outChildrenSurfaceType = MapSurfaceType::Undefined; for (;;) { const auto tag = cis->ReadTag(); switch (gpb::internal::WireFormatLite::GetTagFieldNumber(tag)) { case 0: if (!ObfReaderUtilities::reachedDataEnd(cis)) return; return; case OBF::OsmAndMapIndex_MapDataBox::kBoxesFieldNumber: { const auto length = ObfReaderUtilities::readBigEndianInt(cis); const auto offset = cis->CurrentPosition(); const auto oldLimit = cis->PushLimit(length); const std::shared_ptr<ObfMapSectionLevelTreeNode> childNode(new ObfMapSectionLevelTreeNode(treeNode->level)); childNode->surfaceType = treeNode->surfaceType; childNode->offset = offset; childNode->length = length; readTreeNode(reader, section, treeNode->area31, childNode); ObfReaderUtilities::ensureAllDataWasRead(cis); cis->PopLimit(oldLimit); // Update metric if (metric) metric->visitedNodes++; if (bbox31) { const auto shouldSkip = !bbox31->contains(childNode->area31) && !childNode->area31.contains(*bbox31) && !bbox31->intersects(childNode->area31); if (shouldSkip) break; } // Update metric if (metric) metric->acceptedNodes++; if (nodesWithData && childNode->dataOffset > 0) nodesWithData->push_back(childNode); auto subchildrenSurfaceType = MapSurfaceType::Undefined; if (childNode->hasChildrenDataBoxes) { cis->Seek(childNode->offset); const auto oldLimit = cis->PushLimit(childNode->length); cis->Skip(childNode->firstDataBoxInnerOffset); readTreeNodeChildren(reader, section, childNode, subchildrenSurfaceType, nodesWithData, bbox31, queryController, metric); ObfReaderUtilities::ensureAllDataWasRead(cis); cis->PopLimit(oldLimit); } const auto surfaceTypeToMerge = (subchildrenSurfaceType != MapSurfaceType::Undefined) ? subchildrenSurfaceType : childNode->surfaceType; if (surfaceTypeToMerge != MapSurfaceType::Undefined) { if (outChildrenSurfaceType == MapSurfaceType::Undefined) outChildrenSurfaceType = surfaceTypeToMerge; else if (outChildrenSurfaceType != surfaceTypeToMerge) outChildrenSurfaceType = MapSurfaceType::Mixed; } break; } default: ObfReaderUtilities::skipUnknownField(cis, tag); break; } } }
void OsmAnd::ObfMapSectionReader_P::readTreeNodeChildren( const std::unique_ptr<ObfReader_P>& reader, const std::shared_ptr<const ObfMapSectionInfo>& section, const std::shared_ptr<ObfMapSectionLevelTreeNode>& treeNode, MapFoundationType& foundation, QList< std::shared_ptr<ObfMapSectionLevelTreeNode> >* nodesWithData, const AreaI* bbox31, const IQueryController* const controller, ObfMapSectionReader_Metrics::Metric_loadMapObjects* const metric) { auto cis = reader->_codedInputStream.get(); foundation = MapFoundationType::Undefined; for(;;) { auto tag = cis->ReadTag(); switch(gpb::internal::WireFormatLite::GetTagFieldNumber(tag)) { case 0: return; case OBF::OsmAndMapIndex_MapDataBox::kBoxesFieldNumber: { auto length = ObfReaderUtilities::readBigEndianInt(cis); auto offset = cis->CurrentPosition(); auto oldLimit = cis->PushLimit(length); std::shared_ptr<ObfMapSectionLevelTreeNode> childNode(new ObfMapSectionLevelTreeNode(treeNode->level)); childNode->_foundation = treeNode->_foundation; childNode->_offset = offset; childNode->_length = length; readTreeNode(reader, section, treeNode->_area31, childNode); // Update metric if(metric) metric->visitedNodes++; if(bbox31) { const auto shouldSkip = !bbox31->contains(childNode->_area31) && !childNode->_area31.contains(*bbox31) && !bbox31->intersects(childNode->_area31); if(shouldSkip) { cis->Skip(cis->BytesUntilLimit()); cis->PopLimit(oldLimit); break; } } cis->PopLimit(oldLimit); // Update metric if(metric) metric->acceptedNodes++; if(nodesWithData && childNode->_dataOffset > 0) nodesWithData->push_back(childNode); auto childrenFoundation = MapFoundationType::Undefined; if(childNode->_childrenInnerOffset > 0) { cis->Seek(offset); oldLimit = cis->PushLimit(length); cis->Skip(childNode->_childrenInnerOffset); readTreeNodeChildren(reader, section, childNode, childrenFoundation, nodesWithData, bbox31, controller, metric); assert(cis->BytesUntilLimit() == 0); cis->PopLimit(oldLimit); } const auto foundationToMerge = (childrenFoundation != MapFoundationType::Undefined) ? childrenFoundation : childNode->_foundation; if(foundationToMerge != MapFoundationType::Undefined) { if(foundation == MapFoundationType::Undefined) foundation = foundationToMerge; else if(foundation != foundationToMerge) foundation = MapFoundationType::Mixed; } } break; default: ObfReaderUtilities::skipUnknownField(cis, tag); break; } } }