void OsmAnd::ObfMapSectionReader_P::readTreeNodeChildren(
const std::unique_ptr<ObfReader_P>& reader, const std::shared_ptr<const ObfMapSectionInfo>& section,
const std::shared_ptr<ObfMapSectionLevelTreeNode>& treeNode,
MapFoundationType& foundation,
QList< std::shared_ptr<ObfMapSectionLevelTreeNode> >* nodesWithData,
const AreaI* bbox31,
IQueryController* controller)
{
auto cis = reader->_codedInputStream.get();
for(;;)
{
auto tag = cis->ReadTag();
switch(gpb::internal::WireFormatLite::GetTagFieldNumber(tag))
{
case 0:
return;
case OBF::OsmAndMapIndex_MapDataBox::kBoxesFieldNumber:
{
auto length = ObfReaderUtilities::readBigEndianInt(cis);
auto offset = cis->CurrentPosition();
auto oldLimit = cis->PushLimit(length);
std::shared_ptr<ObfMapSectionLevelTreeNode> childNode(new ObfMapSectionLevelTreeNode());
childNode->_foundation = treeNode->_foundation;
childNode->_offset = offset;
childNode->_length = length;
readTreeNode(reader, section, treeNode->_area31, childNode);
if(bbox31 && !bbox31->intersects(childNode->_area31))
{
cis->Skip(cis->BytesUntilLimit());
cis->PopLimit(oldLimit);
break;
}
cis->PopLimit(oldLimit);
if(childNode->_foundation != MapFoundationType::Undefined)
{
if(foundation == MapFoundationType::Undefined)
foundation = childNode->_foundation;
else if(foundation != childNode->_foundation)
foundation = MapFoundationType::Mixed;
}
if(nodesWithData && childNode->_dataOffset > 0)
nodesWithData->push_back(childNode);
cis->Seek(offset);
oldLimit = cis->PushLimit(length);
cis->Skip(childNode->_childrenInnerOffset);
readTreeNodeChildren(reader, section, childNode, foundation, nodesWithData, bbox31, controller);
assert(cis->BytesUntilLimit() == 0);
cis->PopLimit(oldLimit);
}
break;
default:
ObfReaderUtilities::skipUnknownField(cis, tag);
break;
}
}
}
void OsmAnd::ObfMapSectionReader_P::loadMapObjects(
const ObfReader_P& reader,
const std::shared_ptr<const ObfMapSectionInfo>& section,
ZoomLevel zoom,
const AreaI* bbox31,
QList< std::shared_ptr<const OsmAnd::BinaryMapObject> >* resultOut,
MapSurfaceType* outBBoxOrSectionSurfaceType,
const ObfMapSectionReader::FilterByIdFunction filterById,
const VisitorFunction visitor,
DataBlocksCache* cache,
QList< std::shared_ptr<const DataBlock> >* outReferencedCacheEntries,
const std::shared_ptr<const IQueryController>& queryController,
ObfMapSectionReader_Metrics::Metric_loadMapObjects* const metric)
{
const auto cis = reader.getCodedInputStream().get();
const auto filterReadById =
[filterById, zoom]
(const std::shared_ptr<const ObfMapSectionInfo>& section,
const ObfObjectId mapObjectId,
const AreaI& bbox,
const ZoomLevel firstZoomLevel,
const ZoomLevel lastZoomLevel) -> bool
{
return filterById(section, mapObjectId, bbox, firstZoomLevel, lastZoomLevel, zoom);
};
// Ensure encoding/decoding rules are read
if (section->_p->_attributeMappingLoaded.loadAcquire() == 0)
{
QMutexLocker scopedLocker(§ion->_p->_attributeMappingLoadMutex);
if (!section->_p->_attributeMapping)
{
// Read encoding/decoding rules
cis->Seek(section->offset);
auto oldLimit = cis->PushLimit(section->length);
const std::shared_ptr<ObfMapSectionAttributeMapping> attributeMapping(new ObfMapSectionAttributeMapping());
readAttributeMapping(reader, section, attributeMapping);
section->_p->_attributeMapping = attributeMapping;
cis->PopLimit(oldLimit);
section->_p->_attributeMappingLoaded.storeRelease(1);
}
}
ObfMapSectionReader_Metrics::Metric_loadMapObjects localMetric;
auto bboxOrSectionSurfaceType = MapSurfaceType::Undefined;
if (outBBoxOrSectionSurfaceType)
*outBBoxOrSectionSurfaceType = bboxOrSectionSurfaceType;
QList< std::shared_ptr<const DataBlock> > danglingReferencedCacheEntries;
for (const auto& mapLevel : constOf(section->levels))
{
// Update metric
if (metric)
metric->visitedLevels++;
if (mapLevel->minZoom > zoom || mapLevel->maxZoom < zoom)
continue;
if (bbox31)
{
const Stopwatch bboxLevelCheckStopwatch(metric != nullptr);
const auto shouldSkip =
!bbox31->contains(mapLevel->area31) &&
!mapLevel->area31.contains(*bbox31) &&
!bbox31->intersects(mapLevel->area31);
if (metric)
metric->elapsedTimeForLevelsBbox += bboxLevelCheckStopwatch.elapsed();
if (shouldSkip)
continue;
}
const Stopwatch treeNodesStopwatch(metric != nullptr);
if (metric)
metric->acceptedLevels++;
// If there are no tree nodes in map level, it means they are not loaded.
// Since loading may be called from multiple threads, loading of root nodes needs synchronization
// Ensure encoding/decoding rules are read
if (mapLevel->_p->_rootNodesLoaded.loadAcquire() == 0)
{
QMutexLocker scopedLocker(&mapLevel->_p->_rootNodesLoadMutex);
if (!mapLevel->_p->_rootNodes)
{
cis->Seek(mapLevel->offset);
auto oldLimit = cis->PushLimit(mapLevel->length);
cis->Skip(mapLevel->firstDataBoxInnerOffset);
const std::shared_ptr< QList< std::shared_ptr<const ObfMapSectionLevelTreeNode> > > rootNodes(
new QList< std::shared_ptr<const ObfMapSectionLevelTreeNode> >());
readMapLevelTreeNodes(reader, section, mapLevel, *rootNodes);
mapLevel->_p->_rootNodes = rootNodes;
cis->PopLimit(oldLimit);
mapLevel->_p->_rootNodesLoaded.storeRelease(1);
}
}
// Collect tree nodes with data
QList< std::shared_ptr<const ObfMapSectionLevelTreeNode> > treeNodesWithData;
for (const auto& rootNode : constOf(*mapLevel->_p->_rootNodes))
{
// Update metric
if (metric)
metric->visitedNodes++;
if (bbox31)
{
const Stopwatch bboxNodeCheckStopwatch(metric != nullptr);
const auto shouldSkip =
!bbox31->contains(rootNode->area31) &&
!rootNode->area31.contains(*bbox31) &&
!bbox31->intersects(rootNode->area31);
// Update metric
if (metric)
metric->elapsedTimeForNodesBbox += bboxNodeCheckStopwatch.elapsed();
if (shouldSkip)
continue;
}
// Update metric
if (metric)
metric->acceptedNodes++;
if (rootNode->dataOffset > 0)
treeNodesWithData.push_back(rootNode);
auto rootSubnodesSurfaceType = MapSurfaceType::Undefined;
if (rootNode->hasChildrenDataBoxes)
{
cis->Seek(rootNode->offset);
auto oldLimit = cis->PushLimit(rootNode->length);
cis->Skip(rootNode->firstDataBoxInnerOffset);
readTreeNodeChildren(reader, section, rootNode, rootSubnodesSurfaceType, &treeNodesWithData, bbox31, queryController, metric);
ObfReaderUtilities::ensureAllDataWasRead(cis);
cis->PopLimit(oldLimit);
}
const auto surfaceTypeToMerge = (rootSubnodesSurfaceType != MapSurfaceType::Undefined) ? rootSubnodesSurfaceType : rootNode->surfaceType;
if (surfaceTypeToMerge != MapSurfaceType::Undefined)
{
if (bboxOrSectionSurfaceType == MapSurfaceType::Undefined)
bboxOrSectionSurfaceType = surfaceTypeToMerge;
else if (bboxOrSectionSurfaceType != surfaceTypeToMerge)
bboxOrSectionSurfaceType = MapSurfaceType::Mixed;
}
}
// Sort blocks by data offset to force forward-only seeking
std::sort(treeNodesWithData,
[]
(const std::shared_ptr<const ObfMapSectionLevelTreeNode>& l, const std::shared_ptr<const ObfMapSectionLevelTreeNode>& r) -> bool
{
return l->dataOffset < r->dataOffset;
});
// Update metric
const Stopwatch mapObjectsStopwatch(metric != nullptr);
if (metric)
metric->elapsedTimeForNodes += treeNodesStopwatch.elapsed();
// Read map objects from their blocks
for (const auto& treeNode : constOf(treeNodesWithData))
{
if (queryController && queryController->isAborted())
break;
DataBlockId blockId;
blockId.sectionRuntimeGeneratedId = section->runtimeGeneratedId;
blockId.offset = treeNode->dataOffset;
if (cache && cache->shouldCacheBlock(blockId, treeNode->area31, bbox31))
{
// In case cache is provided, read and cache
const auto levelZooms = Utilities::enumerateZoomLevels(treeNode->level->minZoom, treeNode->level->maxZoom);
std::shared_ptr<const DataBlock> dataBlock;
std::shared_ptr<const DataBlock> sharedBlockReference;
proper::shared_future< std::shared_ptr<const DataBlock> > futureSharedBlockReference;
if (cache->obtainReferenceOrFutureReferenceOrMakePromise(blockId, zoom, levelZooms, sharedBlockReference, futureSharedBlockReference))
{
// Got reference or future reference
// Update metric
if (metric)
metric->mapObjectsBlocksReferenced++;
if (sharedBlockReference)
{
// Ok, this block was already loaded, just use it
dataBlock = sharedBlockReference;
}
else
{
// Wait until it will be loaded
dataBlock = futureSharedBlockReference.get();
}
}
else
{
// Made a promise, so load entire block into temporary storage
QList< std::shared_ptr<const BinaryMapObject> > mapObjects;
cis->Seek(treeNode->dataOffset);
gpb::uint32 length;
cis->ReadVarint32(&length);
const auto oldLimit = cis->PushLimit(length);
readMapObjectsBlock(
reader,
section,
treeNode,
&mapObjects,
nullptr,
nullptr,
nullptr,
nullptr,
metric ? &localMetric : nullptr);
ObfReaderUtilities::ensureAllDataWasRead(cis);
cis->PopLimit(oldLimit);
// Update metric
if (metric)
metric->mapObjectsBlocksRead++;
// Create a data block and share it
dataBlock.reset(new DataBlock(blockId, treeNode->area31, treeNode->surfaceType, mapObjects));
cache->fulfilPromiseAndReference(blockId, levelZooms, dataBlock);
}
if (outReferencedCacheEntries)
outReferencedCacheEntries->push_back(dataBlock);
else
danglingReferencedCacheEntries.push_back(dataBlock);
// Process data block
for (const auto& mapObject : constOf(dataBlock->mapObjects))
{
//////////////////////////////////////////////////////////////////////////
//if (mapObject->id.getOsmId() == 49048972u)
//{
// if (bbox31 && *bbox31 == Utilities::tileBoundingBox31(TileId::fromXY(1052, 673), ZoomLevel11))
// {
// const auto t = mapObject->toString();
// int i = 5;
// }
//}
//////////////////////////////////////////////////////////////////////////
if (metric)
metric->visitedMapObjects++;
if (bbox31)
{
const auto shouldNotSkip =
mapObject->bbox31.contains(*bbox31) ||
bbox31->intersects(mapObject->bbox31);
if (!shouldNotSkip)
continue;
}
// Check if map object is desired
const auto shouldReject = filterById && !filterById(
section,
mapObject->id,
mapObject->bbox31,
mapObject->level->minZoom,
mapObject->level->maxZoom,
zoom);
if (shouldReject)
continue;
if (!visitor || visitor(mapObject))
{
if (metric)
metric->acceptedMapObjects++;
if (resultOut)
resultOut->push_back(qMove(mapObject));
}
}
}
else
{
// In case there's no cache, simply read
cis->Seek(treeNode->dataOffset);
gpb::uint32 length;
cis->ReadVarint32(&length);
const auto oldLimit = cis->PushLimit(length);
readMapObjectsBlock(
reader,
section,
treeNode,
resultOut,
bbox31,
filterById != nullptr ? filterReadById : FilterReadingByIdFunction(),
visitor,
queryController,
metric);
ObfReaderUtilities::ensureAllDataWasRead(cis);
cis->PopLimit(oldLimit);
// Update metric
if (metric)
metric->mapObjectsBlocksRead++;
}
// Update metric
if (metric)
metric->mapObjectsBlocksProcessed++;
}
// Update metric
if (metric)
metric->elapsedTimeForMapObjectsBlocks += mapObjectsStopwatch.elapsed();
}
// In case cache was supplied, but referenced cache entries output collection was not specified,
// release all dangling references
if (cache && !outReferencedCacheEntries)
{
for (auto& referencedCacheEntry : danglingReferencedCacheEntries)
cache->releaseReference(referencedCacheEntry->id, zoom, referencedCacheEntry);
danglingReferencedCacheEntries.clear();
}
if (outBBoxOrSectionSurfaceType)
*outBBoxOrSectionSurfaceType = bboxOrSectionSurfaceType;
// In case cache was used, and metric was requested, some values must be taken from different parts
if (cache && metric)
{
metric->elapsedTimeForOnlyAcceptedMapObjects += localMetric.elapsedTimeForOnlyAcceptedMapObjects;
}
}
void OsmAnd::ObfMapSectionReader_P::readTreeNodeChildren(
const ObfReader_P& reader,
const std::shared_ptr<const ObfMapSectionInfo>& section,
const std::shared_ptr<const ObfMapSectionLevelTreeNode>& treeNode,
MapSurfaceType& outChildrenSurfaceType,
QList< std::shared_ptr<const ObfMapSectionLevelTreeNode> >* nodesWithData,
const AreaI* bbox31,
const std::shared_ptr<const IQueryController>& queryController,
ObfMapSectionReader_Metrics::Metric_loadMapObjects* const metric)
{
const auto cis = reader.getCodedInputStream().get();
outChildrenSurfaceType = MapSurfaceType::Undefined;
for (;;)
{
const auto tag = cis->ReadTag();
switch (gpb::internal::WireFormatLite::GetTagFieldNumber(tag))
{
case 0:
if (!ObfReaderUtilities::reachedDataEnd(cis))
return;
return;
case OBF::OsmAndMapIndex_MapDataBox::kBoxesFieldNumber:
{
const auto length = ObfReaderUtilities::readBigEndianInt(cis);
const auto offset = cis->CurrentPosition();
const auto oldLimit = cis->PushLimit(length);
const std::shared_ptr<ObfMapSectionLevelTreeNode> childNode(new ObfMapSectionLevelTreeNode(treeNode->level));
childNode->surfaceType = treeNode->surfaceType;
childNode->offset = offset;
childNode->length = length;
readTreeNode(reader, section, treeNode->area31, childNode);
ObfReaderUtilities::ensureAllDataWasRead(cis);
cis->PopLimit(oldLimit);
// Update metric
if (metric)
metric->visitedNodes++;
if (bbox31)
{
const auto shouldSkip =
!bbox31->contains(childNode->area31) &&
!childNode->area31.contains(*bbox31) &&
!bbox31->intersects(childNode->area31);
if (shouldSkip)
break;
}
// Update metric
if (metric)
metric->acceptedNodes++;
if (nodesWithData && childNode->dataOffset > 0)
nodesWithData->push_back(childNode);
auto subchildrenSurfaceType = MapSurfaceType::Undefined;
if (childNode->hasChildrenDataBoxes)
{
cis->Seek(childNode->offset);
const auto oldLimit = cis->PushLimit(childNode->length);
cis->Skip(childNode->firstDataBoxInnerOffset);
readTreeNodeChildren(reader, section, childNode, subchildrenSurfaceType, nodesWithData, bbox31, queryController, metric);
ObfReaderUtilities::ensureAllDataWasRead(cis);
cis->PopLimit(oldLimit);
}
const auto surfaceTypeToMerge = (subchildrenSurfaceType != MapSurfaceType::Undefined) ? subchildrenSurfaceType : childNode->surfaceType;
if (surfaceTypeToMerge != MapSurfaceType::Undefined)
{
if (outChildrenSurfaceType == MapSurfaceType::Undefined)
outChildrenSurfaceType = surfaceTypeToMerge;
else if (outChildrenSurfaceType != surfaceTypeToMerge)
outChildrenSurfaceType = MapSurfaceType::Mixed;
}
break;
}
default:
ObfReaderUtilities::skipUnknownField(cis, tag);
break;
}
}
}
void OsmAnd::ObfMapSectionReader_P::readTreeNodeChildren(
const std::unique_ptr<ObfReader_P>& reader, const std::shared_ptr<const ObfMapSectionInfo>& section,
const std::shared_ptr<ObfMapSectionLevelTreeNode>& treeNode,
MapFoundationType& foundation,
QList< std::shared_ptr<ObfMapSectionLevelTreeNode> >* nodesWithData,
const AreaI* bbox31,
const IQueryController* const controller,
ObfMapSectionReader_Metrics::Metric_loadMapObjects* const metric)
{
auto cis = reader->_codedInputStream.get();
foundation = MapFoundationType::Undefined;
for(;;)
{
auto tag = cis->ReadTag();
switch(gpb::internal::WireFormatLite::GetTagFieldNumber(tag))
{
case 0:
return;
case OBF::OsmAndMapIndex_MapDataBox::kBoxesFieldNumber:
{
auto length = ObfReaderUtilities::readBigEndianInt(cis);
auto offset = cis->CurrentPosition();
auto oldLimit = cis->PushLimit(length);
std::shared_ptr<ObfMapSectionLevelTreeNode> childNode(new ObfMapSectionLevelTreeNode(treeNode->level));
childNode->_foundation = treeNode->_foundation;
childNode->_offset = offset;
childNode->_length = length;
readTreeNode(reader, section, treeNode->_area31, childNode);
// Update metric
if(metric)
metric->visitedNodes++;
if(bbox31)
{
const auto shouldSkip =
!bbox31->contains(childNode->_area31) &&
!childNode->_area31.contains(*bbox31) &&
!bbox31->intersects(childNode->_area31);
if(shouldSkip)
{
cis->Skip(cis->BytesUntilLimit());
cis->PopLimit(oldLimit);
break;
}
}
cis->PopLimit(oldLimit);
// Update metric
if(metric)
metric->acceptedNodes++;
if(nodesWithData && childNode->_dataOffset > 0)
nodesWithData->push_back(childNode);
auto childrenFoundation = MapFoundationType::Undefined;
if(childNode->_childrenInnerOffset > 0)
{
cis->Seek(offset);
oldLimit = cis->PushLimit(length);
cis->Skip(childNode->_childrenInnerOffset);
readTreeNodeChildren(reader, section, childNode, childrenFoundation, nodesWithData, bbox31, controller, metric);
assert(cis->BytesUntilLimit() == 0);
cis->PopLimit(oldLimit);
}
const auto foundationToMerge = (childrenFoundation != MapFoundationType::Undefined) ? childrenFoundation : childNode->_foundation;
if(foundationToMerge != MapFoundationType::Undefined)
{
if(foundation == MapFoundationType::Undefined)
foundation = foundationToMerge;
else if(foundation != foundationToMerge)
foundation = MapFoundationType::Mixed;
}
}
break;
default:
ObfReaderUtilities::skipUnknownField(cis, tag);
break;
}
}
}
