void Preprocess::ProcessRelation(const TypeConfig& typeConfig,
const OSMId& id,
const std::vector<RawRelation::Member>& members,
const std::map<TagId,std::string>& tagMap)
{
RawRelation relation;
TypeId type;
if (id<lastRelationId) {
relationSortingError=true;
}
relation.SetId(id);
relation.members=members;
typeConfig.GetRelationTypeId(tagMap,type);
typeConfig.ResolveTags(tagMap,relation.tags);
relation.SetType(type);
relation.Write(relationWriter);
relationCount++;
lastRelationId=id;
}
bool RawNode::Read(const TypeConfig& typeConfig,
FileScanner& scanner)
{
if (!scanner.ReadNumber(id)) {
return false;
}
TypeId typeId;
if (!scanner.ReadTypeId(typeId,
typeConfig.GetNodeTypeIdBytes())) {
return false;
}
TypeInfoRef type=typeConfig.GetNodeTypeInfo(typeId);
featureValueBuffer.SetType(type);
if (!type->GetIgnore()) {
if (!featureValueBuffer.Read(scanner)) {
return false;
}
}
if (!scanner.ReadCoord(coords)) {
return false;
}
return !scanner.HasError();
}
bool Way::ReadOptimized(const TypeConfig& typeConfig,
FileScanner& scanner)
{
if (!scanner.GetPos(fileOffset)) {
return false;
}
TypeId typeId;
scanner.ReadTypeId(typeId,
typeConfig.GetWayTypeIdBytes());
TypeInfoRef type=typeConfig.GetWayTypeInfo(typeId);
featureValueBuffer.SetType(type);
if (!featureValueBuffer.Read(scanner)) {
return false;
}
if (!scanner.Read(nodes)) {
return false;
}
return !scanner.HasError();
}
bool Area::WriteOptimized(const TypeConfig& typeConfig,
FileWriter& writer) const
{
std::vector<Ring>::const_iterator ring=rings.begin();
bool multipleRings=rings.size()>1;
// Outer ring
writer.WriteTypeId(ring->GetType()->GetAreaId(),
typeConfig.GetAreaTypeIdBytes());
if (!ring->featureValueBuffer.Write(writer,
multipleRings)) {
return false;
}
if (multipleRings) {
writer.WriteNumber((uint32_t)(rings.size()-1));
}
if (!writer.Write(ring->nodes)) {
return false;
}
++ring;
// Potential additional rings
while (ring!=rings.end()) {
writer.WriteTypeId(ring->GetType()->GetAreaId(),
typeConfig.GetAreaTypeIdBytes());
if (ring->GetType()->GetAreaId()!=typeIgnore) {
if (!ring->featureValueBuffer.Write(writer)) {
return false;
}
}
writer.Write(ring->ring);
if (!writer.Write(ring->nodes)) {
return false;
}
++ring;
}
return !writer.HasError();
}
void PreprocessPBF::ReadNodes(const TypeConfig& typeConfig,
const PBF::PrimitiveBlock& block,
const PBF::PrimitiveGroup& group,
PreprocessorCallback::RawBlockData& data)
{
data.nodeData.reserve(data.nodeData.size()+group.nodes_size());
for (int n=0; n<group.nodes_size(); n++) {
PreprocessorCallback::RawNodeData nodeData;
const PBF::Node &inputNode=group.nodes(n);
nodeData.id=inputNode.id();
nodeData.coord.Set((inputNode.lat()*block.granularity()+block.lat_offset())/NANO,
(inputNode.lon()*block.granularity()+block.lon_offset())/NANO);
tagMap.clear();
for (int t=0; t<inputNode.keys_size(); t++) {
TagId id=typeConfig.GetTagId(block.stringtable().s(inputNode.keys(t)));
if (id!=tagIgnore) {
nodeData.tags[id]=block.stringtable().s(inputNode.vals(t));
}
}
data.nodeData.push_back(std::move(nodeData));
}
}
bool Way::Write(const TypeConfig& typeConfig,
FileWriter& writer) const
{
assert(!nodes.empty());
writer.WriteTypeId(featureValueBuffer.GetType()->GetWayId(),
typeConfig.GetWayTypeIdBytes());
if (!featureValueBuffer.Write(writer)) {
return false;
}
if (!writer.Write(nodes)) {
return false;
}
if (featureValueBuffer.GetType()->CanRoute() ||
featureValueBuffer.GetType()->GetOptimizeLowZoom()) {
if (!WriteIds(writer)) {
return false;
}
}
return !writer.HasError();
}
void PreprocessPBF::ReadWays(const TypeConfig& typeConfig,
const PBF::PrimitiveBlock& block,
const PBF::PrimitiveGroup& group,
PreprocessorCallback::RawBlockData& data)
{
data.wayData.reserve(data.wayData.size()+group.ways_size());
for (int w=0; w<group.ways_size(); w++) {
PreprocessorCallback::RawWayData wayData;
const PBF::Way &inputWay=group.ways(w);
wayData.id=inputWay.id();
for (int t=0; t<inputWay.keys_size(); t++) {
TagId id=typeConfig.GetTagId(block.stringtable().s(inputWay.keys(t)));
if (id!=tagIgnore) {
wayData.tags[id]=block.stringtable().s(inputWay.vals(t));
}
}
wayData.nodes.reserve(inputWay.refs_size());
OSMId ref=0;
for (int r=0; r<inputWay.refs_size(); r++) {
ref+=inputWay.refs(r);
wayData.nodes.push_back(ref);
}
data.wayData.push_back(std::move(wayData));
}
}
void PreprocessPBF::ReadRelations(const TypeConfig& typeConfig,
const PBF::PrimitiveBlock& block,
const PBF::PrimitiveGroup& group,
PreprocessorCallback::RawBlockData& data)
{
data.relationData.reserve(data.relationData.size()+group.relations_size());
for (int r=0; r<group.relations_size(); r++) {
PreprocessorCallback::RawRelationData relationData;
const PBF::Relation &inputRelation=group.relations(r);
relationData.id=inputRelation.id();
members.clear();
for (int t=0; t<inputRelation.keys_size(); t++) {
TagId id=typeConfig.GetTagId(block.stringtable().s(inputRelation.keys(t)));
if (id!=tagIgnore) {
relationData.tags[id]=block.stringtable().s(inputRelation.vals(t));
}
}
relationData.members.reserve(inputRelation.types_size());
Id ref=0;
for (int m=0; m<inputRelation.types_size(); m++) {
RawRelation::Member member;
switch (inputRelation.types(m)) {
case PBF::Relation::NODE:
member.type=RawRelation::memberNode;
break;
case PBF::Relation::WAY:
member.type=RawRelation::memberWay;
break;
case PBF::Relation::RELATION:
member.type=RawRelation::memberRelation;
break;
}
ref+=inputRelation.memids(m);
member.id=ref;
member.role=block.stringtable().s(inputRelation.roles_sid(m));
relationData.members.push_back(member);
}
data.relationData.push_back(std::move(relationData));
}
}
void OptimizeAreasLowZoomGenerator::GetAreaTypesToOptimize(const TypeConfig& typeConfig,
TypeInfoSet& types)
{
types.Clear();
for (auto &type : typeConfig.GetAreaTypes()) {
if (!type->GetIgnore() &&
type->GetOptimizeLowZoom()) {
types.Set(type);
}
}
}
void Preprocess::ProcessNode(const TypeConfig& typeConfig,
const OSMId& id,
const double& lon,
const double& lat,
const std::map<TagId,std::string>& tagMap)
{
RawNode node;
TypeId type=typeIgnore;
FileOffset nodeOffset;
if (id<lastNodeId) {
nodeSortingError=true;
}
typeConfig.GetNodeTypeId(tagMap,type);
if (type!=typeIgnore) {
typeConfig.ResolveTags(tagMap,tags);
nodeWriter.GetPos(nodeOffset);
node.SetId(id);
node.SetType(type);
node.SetCoords(lon,lat);
node.SetTags(tags);
node.Write(nodeWriter);
nodeCount++;
}
else {
nodeOffset=0;
}
StoreCoord(id,
lat,
lon);
lastNodeId=id;
}
void AbstractRoutingProfile::ParametrizeForFoot(const TypeConfig& typeConfig,
double maxSpeed)
{
speeds.clear();
SetVehicle(vehicleFoot);
SetVehicleMaxSpeed(maxSpeed);
for (const auto &type : typeConfig.GetTypes()) {
if (!type->GetIgnore() &&
type->CanRouteFoot()) {
AddType(type,maxSpeed);
}
}
}
void PreprocessPBF::ReadDenseNodes(const TypeConfig& typeConfig,
const PBF::PrimitiveBlock& block,
const PBF::PrimitiveGroup& group,
PreprocessorCallback::RawBlockData& data)
{
const PBF::DenseNodes& dense=group.dense();
Id dId=0;
double dLat=0;
double dLon=0;
int t=0;
data.nodeData.reserve(data.nodeData.size()+dense.id_size());
for (int d=0; d<dense.id_size();d++) {
PreprocessorCallback::RawNodeData nodeData;
dId+=dense.id(d);
dLat+=dense.lat(d);
dLon+=dense.lon(d);
nodeData.id=dId;
nodeData.coord.Set((dLat*block.granularity()+block.lat_offset())/NANO,
(dLon*block.granularity()+block.lon_offset())/NANO);
while (true) {
if (t>=dense.keys_vals_size()) {
break;
}
if (dense.keys_vals(t)==0) {
t++;
break;
}
TagId id=typeConfig.GetTagId(block.stringtable().s(dense.keys_vals(t)));
if (id!=tagIgnore) {
nodeData.tags[id]=block.stringtable().s(dense.keys_vals(t+1));
}
t+=2;
}
data.nodeData.push_back(std::move(nodeData));
}
}
bool RawNode::Write(const TypeConfig& typeConfig,
FileWriter& writer) const
{
writer.WriteNumber(id);
writer.WriteTypeId(featureValueBuffer.GetType()->GetNodeId(),
typeConfig.GetNodeTypeIdBytes());
if (!featureValueBuffer.GetType()->GetIgnore()) {
if (!featureValueBuffer.Write(writer)) {
return false;
}
}
writer.WriteCoord(coords);
return !writer.HasError();
}
bool Node::Read(const TypeConfig& typeConfig,
FileScanner& scanner)
{
if (!scanner.GetPos(fileOffset)) {
return false;
}
uint32_t tmpType;
scanner.ReadNumber(tmpType);
TypeInfoRef type=typeConfig.GetTypeInfo((TypeId)tmpType);
featureValueBuffer.SetType(type);
if (!featureValueBuffer.Read(scanner)) {
return false;
}
scanner.ReadCoord(coords);
return !scanner.HasError();
}
bool OptimizeAreasLowZoomGenerator::HandleAreas(const ImportParameter& parameter,
Progress& progress,
const TypeConfig& typeConfig,
FileWriter& writer,
const TypeInfoSet& types,
std::list<TypeData>& typesData)
{
FileScanner scanner;
// Everything smaller than 2mm should get dropped. Width, height and DPI come from the Nexus 4
double dpi=320.0;
double pixel=2.0/* mm */ * dpi / 25.4 /* inch */;
progress.Info("Minimum visible size in pixel: "+NumberToString((unsigned long)pixel));
try {
scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
AreaDataFile::AREAS_DAT),
FileScanner::Sequential,
parameter.GetWayDataMemoryMaped());
TypeInfoSet typesToProcess(types);
std::vector<std::list<AreaRef> > allAreas(typeConfig.GetTypeCount());
while (true) {
//
// Load type data
//
TypeInfoSet loadedTypes;
if (!GetAreas(typeConfig,
parameter,
progress,
scanner,
typesToProcess,
allAreas,
loadedTypes)) {
return false;
}
typesToProcess.Remove(loadedTypes);
for (const auto& type : loadedTypes) {
progress.SetAction("Optimizing type "+ type->GetName());
for (uint32_t level=parameter.GetOptimizationMinMag();
level<=parameter.GetOptimizationMaxMag();
level++) {
Magnification magnification; // Magnification, we optimize for
std::list<AreaRef> optimizedAreas;
magnification.SetLevel(level);
OptimizeAreas(allAreas[type->GetIndex()],
optimizedAreas,
1280,768,
dpi,
pixel,
magnification,
parameter.GetOptimizationWayMethod());
if (optimizedAreas.empty()) {
progress.Debug("Empty optimization result for level "+NumberToString(level)+", no index generated");
TypeData typeData;
typeData.type=type;
typeData.optLevel=level;
typesData.push_back(typeData);
continue;
}
progress.Info("Optimized from "+NumberToString(allAreas[type->GetIndex()].size())+" to "+NumberToString(optimizedAreas.size())+" areas");
/*
size_t optAreas=optimizedAreas.size();
size_t optRoles=0;
size_t optNodes=0;
for (std::list<AreaRef>::const_iterator a=optimizedAreas.begin();
a!=optimizedAreas.end();
++a) {
AreaRef area=*a;
optRoles+=area->rings.size();
for (size_t r=0; r<area->rings.size(); r++) {
optNodes+=area->rings[r].nodes.size();
}
}*/
/*
std::cout << "Areas: " << origAreas << " => " << optAreas << std::endl;
std::cout << "Roles: " << origRoles << " => " << optRoles << std::endl;
std::cout << "Nodes: " << origNodes << " => " << optNodes << std::endl;*/
TypeData typeData;
typeData.type=type;
typeData.optLevel=level;
GetAreaIndexLevel(parameter,
optimizedAreas,
typeData);
//std::cout << "Resulting index level: " << typeData.indexLevel << ", " << typeData.indexCells << ", " << typeData.indexEntries << std::endl;
FileOffsetFileOffsetMap offsets;
WriteAreas(typeConfig,
writer,
optimizedAreas,
offsets);
if (!WriteAreaBitmap(progress,
writer,
optimizedAreas,
offsets,
typeData)) {
return false;
}
typesData.push_back(typeData);
}
allAreas[type->GetIndex()].clear();
}
if (typesToProcess.Empty()) {
break;
}
}
scanner.Close();
}
catch (IOException& e) {
progress.Error(e.GetDescription());
return false;
}
return true;
}
void Preprocess::ProcessWay(const TypeConfig& typeConfig,
const OSMId& id,
std::vector<OSMId>& nodes,
const std::map<TagId,std::string>& tagMap)
{
TypeId areaType=typeIgnore;
TypeId wayType=typeIgnore;
int isArea=0; // 0==unknown, 1==true, -1==false
std::map<TagId,std::string>::const_iterator areaTag;
std::map<TagId,std::string>::const_iterator naturalTag;
RawWay way;
bool isCoastline=false;
if (id<lastWayId) {
waySortingError=true;
}
way.SetId(id);
areaTag=tagMap.find(typeConfig.tagArea);
if (areaTag==tagMap.end()) {
isArea=0;
}
else if (areaTag->second=="no" ||
areaTag->second=="false" ||
areaTag->second=="0") {
isArea=-1;
}
else {
isArea=1;
}
naturalTag=tagMap.find(typeConfig.tagNatural);
if (naturalTag!=tagMap.end() &&
naturalTag->second=="coastline") {
isCoastline=true;
}
typeConfig.GetWayAreaTypeId(tagMap,wayType,areaType);
typeConfig.ResolveTags(tagMap,tags);
if (isArea==1 &&
areaType==typeIgnore) {
isArea=0;
}
else if (isArea==-1 &&
wayType==typeIgnore) {
isArea=0;
}
if (isArea==0) {
if (wayType!=typeIgnore &&
areaType==typeIgnore) {
isArea=-1;
}
else if (wayType==typeIgnore &&
areaType!=typeIgnore) {
isArea=1;
}
else if (wayType!=typeIgnore &&
areaType!=typeIgnore) {
if (nodes.size()>3 &&
nodes.front()==nodes.back()) {
if (typeConfig.GetTypeInfo(wayType).GetPinWay()) {
isArea=-1;
}
else {
isArea=1;
}
}
else {
isArea=-1;
}
}
}
switch (isArea) {
case 1:
way.SetType(areaType,true);
if (nodes.size()>3 &&
nodes.front()==nodes.back()) {
nodes.pop_back();
}
areaCount++;
break;
case -1:
way.SetType(wayType,false);
wayCount++;
break;
default:
if (nodes.size()>3 &&
nodes.front()==nodes.back()) {
way.SetType(typeIgnore,
true);
nodes.pop_back();
areaCount++;
}
else {
way.SetType(typeIgnore,
false);
wayCount++;
}
}
way.SetNodes(nodes);
way.SetTags(tags);
way.Write(wayWriter);
lastWayId=id;
if (isCoastline) {
RawCoastline coastline;
coastline.SetId(way.GetId());
coastline.SetType(way.IsArea());
coastline.SetNodes(way.GetNodes());
coastline.Write(coastlineWriter);
coastlineCount++;
}
}
bool AreaWayIndexGenerator::Import(const ImportParameter& parameter,
Progress& progress,
const TypeConfig& typeConfig)
{
FileScanner wayScanner;
FileWriter writer;
std::set<TypeId> remainingWayTypes;
std::vector<TypeData> wayTypeData;
size_t level;
size_t maxLevel=0;
wayTypeData.resize(typeConfig.GetTypes().size());
if (!wayScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"ways.dat"),
FileScanner::Sequential,
parameter.GetWayDataMemoryMaped())) {
progress.Error("Cannot open 'ways.dat'");
return false;
}
//
// Scanning distribution
//
progress.SetAction("Scanning level distribution of way types");
for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
if (typeConfig.GetTypeInfo(i).CanBeWay() &&
!typeConfig.GetTypeInfo(i).GetIgnore()) {
remainingWayTypes.insert(i);
}
}
level=parameter.GetAreaWayMinMag();
while (!remainingWayTypes.empty()) {
uint32_t wayCount=0;
std::set<TypeId> currentWayTypes(remainingWayTypes);
double cellWidth=360.0/pow(2.0,(int)level);
double cellHeight=180.0/pow(2.0,(int)level);
std::vector<CoordCountMap> cellFillCount(typeConfig.GetTypes().size());
progress.Info("Scanning Level "+NumberToString(level)+" ("+NumberToString(remainingWayTypes.size())+" types remaining)");
wayScanner.GotoBegin();
if (!wayScanner.Read(wayCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
Way way;
for (uint32_t w=1; w<=wayCount; w++) {
progress.SetProgress(w,wayCount);
if (!way.Read(wayScanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(w)+" of "+
NumberToString(wayCount)+
" in file '"+
wayScanner.GetFilename()+"'");
return false;
}
// Count number of entries per current type and coordinate
if (currentWayTypes.find(way.GetType())==currentWayTypes.end()) {
continue;
}
double minLon;
double maxLon;
double minLat;
double maxLat;
way.GetBoundingBox(minLon,maxLon,minLat,maxLat);
//
// Calculate minimum and maximum tile ids that are covered
// by the way
// Renormated coordinate space (everything is >=0)
//
uint32_t minxc=(uint32_t)floor((minLon+180.0)/cellWidth);
uint32_t maxxc=(uint32_t)floor((maxLon+180.0)/cellWidth);
uint32_t minyc=(uint32_t)floor((minLat+90.0)/cellHeight);
uint32_t maxyc=(uint32_t)floor((maxLat+90.0)/cellHeight);
for (uint32_t y=minyc; y<=maxyc; y++) {
for (uint32_t x=minxc; x<=maxxc; x++) {
cellFillCount[way.GetType()][Pixel(x,y)]++;
}
}
}
// Check if cell fill for current type is in defined limits
for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
if (currentWayTypes.find(i)!=currentWayTypes.end()) {
CalculateStatistics(level,wayTypeData[i],cellFillCount[i]);
if (!FitsIndexCriteria(parameter,
progress,
typeConfig.GetTypeInfo(i),
wayTypeData[i],
cellFillCount[i])) {
currentWayTypes.erase(i);
}
}
}
for (std::set<TypeId>::const_iterator cwt=currentWayTypes.begin();
cwt!=currentWayTypes.end();
cwt++) {
maxLevel=std::max(maxLevel,level);
progress.Info("Type "+typeConfig.GetTypeInfo(*cwt).GetName()+"(" + NumberToString(*cwt)+"), "+NumberToString(wayTypeData[*cwt].indexCells)+" cells, "+NumberToString(wayTypeData[*cwt].indexEntries)+" objects");
remainingWayTypes.erase(*cwt);
}
level++;
}
//
// Writing index file
//
progress.SetAction("Generating 'areaway.idx'");
if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"areaway.idx"))) {
progress.Error("Cannot create 'areaway.idx'");
return false;
}
uint32_t indexEntries=0;
for (size_t i=0; i<typeConfig.GetTypes().size(); i++)
{
if (typeConfig.GetTypeInfo(i).CanBeWay() &&
wayTypeData[i].HasEntries()) {
indexEntries++;
}
}
writer.Write(indexEntries);
for (size_t i=0; i<typeConfig.GetTypes().size(); i++)
{
if (typeConfig.GetTypeInfo(i).CanBeWay() &&
wayTypeData[i].HasEntries()) {
uint8_t dataOffsetBytes=0;
FileOffset bitmapOffset=0;
writer.WriteNumber(typeConfig.GetTypeInfo(i).GetId());
writer.GetPos(wayTypeData[i].indexOffset);
writer.WriteFileOffset(bitmapOffset);
if (wayTypeData[i].HasEntries()) {
writer.Write(dataOffsetBytes);
writer.WriteNumber(wayTypeData[i].indexLevel);
writer.WriteNumber(wayTypeData[i].cellXStart);
writer.WriteNumber(wayTypeData[i].cellXEnd);
writer.WriteNumber(wayTypeData[i].cellYStart);
writer.WriteNumber(wayTypeData[i].cellYEnd);
}
}
}
for (size_t l=parameter.GetAreaWayMinMag(); l<=maxLevel; l++) {
std::set<TypeId> indexTypes;
uint32_t wayCount;
double cellWidth=360.0/pow(2.0,(int)l);
double cellHeight=180.0/pow(2.0,(int)l);
for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
if (typeConfig.GetTypeInfo(i).CanBeWay() &&
wayTypeData[i].HasEntries() &&
wayTypeData[i].indexLevel==l) {
indexTypes.insert(i);
}
}
if (indexTypes.empty()) {
continue;
}
progress.Info("Scanning ways for index level "+NumberToString(l));
std::vector<CoordOffsetsMap> typeCellOffsets(typeConfig.GetTypes().size());
wayScanner.GotoBegin();
if (!wayScanner.Read(wayCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
Way way;
for (uint32_t w=1; w<=wayCount; w++) {
progress.SetProgress(w,wayCount);
FileOffset offset;
wayScanner.GetPos(offset);
if (!way.Read(wayScanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(w)+" of "+
NumberToString(wayCount)+
" in file '"+
wayScanner.GetFilename()+"'");
return false;
}
if (indexTypes.find(way.GetType())==indexTypes.end()) {
continue;
}
double minLon;
double maxLon;
double minLat;
double maxLat;
way.GetBoundingBox(minLon,maxLon,minLat,maxLat);
//
// Calculate minimum and maximum tile ids that are covered
// by the way
// Renormated coordinate space (everything is >=0)
//
uint32_t minxc=(uint32_t)floor((minLon+180.0)/cellWidth);
uint32_t maxxc=(uint32_t)floor((maxLon+180.0)/cellWidth);
uint32_t minyc=(uint32_t)floor((minLat+90.0)/cellHeight);
uint32_t maxyc=(uint32_t)floor((maxLat+90.0)/cellHeight);
for (uint32_t y=minyc; y<=maxyc; y++) {
for (uint32_t x=minxc; x<=maxxc; x++) {
typeCellOffsets[way.GetType()][Pixel(x,y)].push_back(offset);
}
}
}
for (std::set<TypeId>::const_iterator type=indexTypes.begin();
type!=indexTypes.end();
++type) {
if (!WriteBitmap(progress,
writer,
typeConfig.GetTypeInfo(*type),
wayTypeData[*type],
typeCellOffsets[*type])) {
return false;
}
}
}
return !writer.HasError() && writer.Close();
}
bool AreaAreaIndex::GetIndexCell(const TypeConfig& typeConfig,
uint32_t level,
FileOffset offset,
IndexCache::CacheRef& cacheRef) const
{
if (!indexCache.GetEntry(offset,cacheRef)) {
IndexCache::CacheEntry cacheEntry(offset);
cacheRef=indexCache.SetEntry(cacheEntry);
if (!scanner.IsOpen()) {
if (!scanner.Open(datafilename,FileScanner::LowMemRandom,true)) {
log.Error() << "Error while opening '" << scanner.GetFilename() << "' for reading!";
return false;
}
}
scanner.SetPos(offset);
// Read offsets of children if not in the bottom level
if (level<maxLevel) {
for (size_t c=0; c<4; c++) {
if (!scanner.ReadNumber(cacheRef->value.children[c])) {
log.Error() << "Cannot read index data at offset " << offset << " in file '" << scanner.GetFilename() << "'";
return false;
}
}
}
else {
for (size_t c=0; c<4; c++) {
cacheRef->value.children[c]=0;
}
}
// Now read the way offsets by type in this index entry
uint32_t offsetCount;
// Areas
if (!scanner.ReadNumber(offsetCount)) {
log.Error() << "Cannot read index data for level " << level << " at offset " << offset << " in file '" << scanner.GetFilename() << "'";
return false;
}
cacheRef->value.areas.resize(offsetCount);
FileOffset prevOffset=0;
for (size_t c=0; c<offsetCount; c++) {
if (!scanner.ReadTypeId(cacheRef->value.areas[c].type,
typeConfig.GetAreaTypeIdBytes())) {
log.Error() << "Cannot read index data for level " << level << " at offset " << offset << " in file '" << scanner.GetFilename() << "'";
return false;
}
if (!scanner.ReadNumber(cacheRef->value.areas[c].offset)) {
log.Error() << "Cannot read index data for level " << level << " at offset " << offset << " in file '" << scanner.GetFilename() << "'";
return false;
}
cacheRef->value.areas[c].offset+=prevOffset;
prevOffset=cacheRef->value.areas[c].offset;
}
}
return true;
}
bool AreaNodeIndexGenerator::Import(const ImportParameter& parameter,
Progress& progress,
const TypeConfig& typeConfig)
{
FileScanner nodeScanner;
FileWriter writer;
std::set<TypeId> remainingNodeTypes; //! Set of types we still must process
std::vector<TypeData> nodeTypeData;
size_t level;
size_t maxLevel=0;
nodeTypeData.resize(typeConfig.GetTypes().size());
if (!nodeScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"nodes.dat"),
FileScanner::Sequential,
true)) {
progress.Error("Cannot open 'nodes.dat'");
return false;
}
//
// Scanning distribution
//
progress.SetAction("Scanning level distribution of node types");
// Initially we must process all types that represents nodes and that should
// not be ignored
for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
if (typeConfig.GetTypeInfo(i).CanBeNode() &&
!typeConfig.GetTypeInfo(i).GetIgnore()) {
remainingNodeTypes.insert(i);
}
}
level=parameter.GetAreaNodeMinMag();
while (!remainingNodeTypes.empty()) {
uint32_t nodeCount=0;
std::set<TypeId> currentNodeTypes(remainingNodeTypes);
double cellWidth=360.0/pow(2.0,(int)level);
double cellHeight=180.0/pow(2.0,(int)level);
std::vector<std::map<Pixel,size_t> > cellFillCount;
cellFillCount.resize(typeConfig.GetTypes().size());
progress.Info("Scanning Level "+NumberToString(level)+" ("+NumberToString(remainingNodeTypes.size())+" types still to process)");
nodeScanner.GotoBegin();
if (!nodeScanner.Read(nodeCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
for (uint32_t n=1; n<=nodeCount; n++) {
progress.SetProgress(n,nodeCount);
FileOffset offset;
Node node;
nodeScanner.GetPos(offset);
if (!node.Read(nodeScanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(n)+" of "+
NumberToString(nodeCount)+
" in file '"+
nodeScanner.GetFilename()+"'");
return false;
}
// If we still need to handle this type,
// count number of entries per type and tile cell
if (currentNodeTypes.find(node.GetType())!=currentNodeTypes.end()) {
uint32_t xc=(uint32_t)floor((node.GetLon()+180.0)/cellWidth);
uint32_t yc=(uint32_t)floor((node.GetLat()+90.0)/cellHeight);
cellFillCount[node.GetType()][Pixel(xc,yc)]++;
}
}
// Check statistics for each type
// If statistics are within goal limits, use this level
// for this type (else try again with the next higher level)
for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
if (currentNodeTypes.find(i)!=currentNodeTypes.end()) {
size_t entryCount=0;
size_t max=0;
nodeTypeData[i].indexLevel=(uint32_t)level;
nodeTypeData[i].indexCells=cellFillCount[i].size();
nodeTypeData[i].indexEntries=0;
if (!cellFillCount[i].empty()) {
nodeTypeData[i].cellXStart=cellFillCount[i].begin()->first.x;
nodeTypeData[i].cellYStart=cellFillCount[i].begin()->first.y;
nodeTypeData[i].cellXEnd=nodeTypeData[i].cellXStart;
nodeTypeData[i].cellYEnd=nodeTypeData[i].cellYStart;
for (std::map<Pixel,size_t>::const_iterator cell=cellFillCount[i].begin();
cell!=cellFillCount[i].end();
++cell) {
nodeTypeData[i].indexEntries+=cell->second;
nodeTypeData[i].cellXStart=std::min(nodeTypeData[i].cellXStart,cell->first.x);
nodeTypeData[i].cellXEnd=std::max(nodeTypeData[i].cellXEnd,cell->first.x);
nodeTypeData[i].cellYStart=std::min(nodeTypeData[i].cellYStart,cell->first.y);
nodeTypeData[i].cellYEnd=std::max(nodeTypeData[i].cellYEnd,cell->first.y);
}
}
nodeTypeData[i].cellXCount=nodeTypeData[i].cellXEnd-nodeTypeData[i].cellXStart+1;
nodeTypeData[i].cellYCount=nodeTypeData[i].cellYEnd-nodeTypeData[i].cellYStart+1;
// Count absolute number of entries
for (std::map<Pixel,size_t>::const_iterator cell=cellFillCount[i].begin();
cell!=cellFillCount[i].end();
++cell) {
entryCount+=cell->second;
max=std::max(max,cell->second);
}
// Average number of entries per tile cell
double average=entryCount*1.0/cellFillCount[i].size();
// If we do not have any entries, we store it now
if (cellFillCount[i].empty()) {
continue;
}
// If the fill rate of the index is too low, we use this index level anyway
if (nodeTypeData[i].indexCells/(1.0*nodeTypeData[i].cellXCount*nodeTypeData[i].cellYCount)<=
parameter.GetAreaNodeIndexMinFillRate()) {
progress.Warning(typeConfig.GetTypeInfo(i).GetName()+" ("+NumberToString(i)+") is not well distributed");
continue;
}
// If average fill size and max fill size for tile cells
// is within limits, store it now.
if (max<=parameter.GetAreaNodeIndexCellSizeMax() &&
average<=parameter.GetAreaNodeIndexCellSizeAverage()) {
continue;
}
// else, we remove it from the list and try again with an higher
// level.
currentNodeTypes.erase(i);
}
}
// Now process all types for this limit, that are within the limits
for (std::set<TypeId>::const_iterator cnt=currentNodeTypes.begin();
cnt!=currentNodeTypes.end();
cnt++) {
maxLevel=std::max(maxLevel,level);
progress.Info("Type "+typeConfig.GetTypeInfo(*cnt).GetName()+"(" + NumberToString(*cnt)+"), "+NumberToString(nodeTypeData[*cnt].indexCells)+" cells, "+NumberToString(nodeTypeData[*cnt].indexEntries)+" objects");
remainingNodeTypes.erase(*cnt);
}
level++;
}
//
// Writing index file
//
progress.SetAction("Generating 'areanode.idx'");
if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"areanode.idx"))) {
progress.Error("Cannot create 'areanode.idx'");
return false;
}
uint32_t indexEntries=0;
// Count number of types in index
for (size_t i=0; i<typeConfig.GetTypes().size(); i++)
{
if (typeConfig.GetTypeInfo(i).CanBeNode() &&
nodeTypeData[i].HasEntries()) {
indexEntries++;
}
}
writer.Write(indexEntries);
// Store index data for each type
for (size_t i=0; i<typeConfig.GetTypes().size(); i++)
{
if (typeConfig.GetTypeInfo(i).CanBeNode() &&
nodeTypeData[i].HasEntries()) {
FileOffset bitmapOffset=0;
uint8_t dataOffsetBytes=0;
writer.WriteNumber(typeConfig.GetTypeInfo(i).GetId());
writer.GetPos(nodeTypeData[i].indexOffset);
writer.WriteFileOffset(bitmapOffset);
writer.Write(dataOffsetBytes);
writer.WriteNumber(nodeTypeData[i].indexLevel);
writer.WriteNumber(nodeTypeData[i].cellXStart);
writer.WriteNumber(nodeTypeData[i].cellXEnd);
writer.WriteNumber(nodeTypeData[i].cellYStart);
writer.WriteNumber(nodeTypeData[i].cellYEnd);
}
}
// Now store index bitmap for each type in increasing level order (why?)
for (size_t l=0; l<=maxLevel; l++) {
std::set<TypeId> indexTypes;
uint32_t nodeCount;
double cellWidth=360.0/pow(2.0,(int)l);
double cellHeight=180.0/pow(2.0,(int)l);
for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
if (typeConfig.GetTypeInfo(i).CanBeNode() &&
nodeTypeData[i].HasEntries() &&
nodeTypeData[i].indexLevel==l) {
indexTypes.insert(i);
}
}
if (indexTypes.empty()) {
continue;
}
progress.Info("Scanning nodes for index level "+NumberToString(l));
std::vector<std::map<Pixel,std::list<FileOffset> > > typeCellOffsets;
typeCellOffsets.resize(typeConfig.GetTypes().size());
nodeScanner.GotoBegin();
if (!nodeScanner.Read(nodeCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
//
// Collect all offsets
//
for (uint32_t n=1; n<=nodeCount; n++) {
progress.SetProgress(n,nodeCount);
FileOffset offset;
Node node;
nodeScanner.GetPos(offset);
if (!node.Read(nodeScanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(n)+" of "+
NumberToString(nodeCount)+
" in file '"+
nodeScanner.GetFilename()+"'");
return false;
}
if (indexTypes.find(node.GetType())!=indexTypes.end()) {
uint32_t xc=(uint32_t)floor((node.GetLon()+180.0)/cellWidth);
uint32_t yc=(uint32_t)floor((node.GetLat()+90.0)/cellHeight);
typeCellOffsets[node.GetType()][Pixel(xc,yc)].push_back(offset);
}
}
//
// Write bitmap
//
for (std::set<TypeId>::const_iterator type=indexTypes.begin();
type!=indexTypes.end();
++type) {
size_t indexEntries=0;
size_t dataSize=0;
char buffer[10];
for (std::map<Pixel,std::list<FileOffset> >::const_iterator cell=typeCellOffsets[*type].begin();
cell!=typeCellOffsets[*type].end();
++cell) {
indexEntries+=cell->second.size();
dataSize+=EncodeNumber(cell->second.size(),buffer);
FileOffset previousOffset=0;
for (std::list<FileOffset>::const_iterator offset=cell->second.begin();
offset!=cell->second.end();
++offset) {
FileOffset data=*offset-previousOffset;
dataSize+=EncodeNumber(data,buffer);
previousOffset=*offset;
}
}
// "+1" because we add +1 to every offset, to generate offset > 0
uint8_t dataOffsetBytes=BytesNeededToAddressFileData(dataSize);
progress.Info("Writing map for "+
typeConfig.GetTypeInfo(*type).GetName()+", "+
NumberToString(typeCellOffsets[*type].size())+" cells, "+
NumberToString(indexEntries)+" entries, "+
ByteSizeToString(1.0*dataOffsetBytes*nodeTypeData[*type].cellXCount*nodeTypeData[*type].cellYCount));
FileOffset bitmapOffset;
if (!writer.GetPos(bitmapOffset)) {
progress.Error("Cannot get type index start position in file");
return false;
}
assert(nodeTypeData[*type].indexOffset!=0);
if (!writer.SetPos(nodeTypeData[*type].indexOffset)) {
progress.Error("Cannot go to type index offset in file");
return false;
}
writer.WriteFileOffset(bitmapOffset);
writer.Write(dataOffsetBytes);
if (!writer.SetPos(bitmapOffset)) {
progress.Error("Cannot go to type index start position in file");
return false;
}
// Write the bitmap with offsets for each cell
// We prefill with zero and only overwrite cells that have data
// So zero means "no data for this cell"
for (size_t i=0; i<nodeTypeData[*type].cellXCount*nodeTypeData[*type].cellYCount; i++) {
FileOffset cellOffset=0;
writer.WriteFileOffset(cellOffset,
dataOffsetBytes);
}
FileOffset dataStartOffset;
if (!writer.GetPos(dataStartOffset)) {
progress.Error("Cannot get start of data section after bitmap");
return false;
}
// Now write the list of offsets of objects for every cell with content
for (std::map<Pixel,std::list<FileOffset> >::const_iterator cell=typeCellOffsets[*type].begin();
cell!=typeCellOffsets[*type].end();
++cell) {
FileOffset bitmapCellOffset=bitmapOffset+
((cell->first.y-nodeTypeData[*type].cellYStart)*nodeTypeData[*type].cellXCount+
cell->first.x-nodeTypeData[*type].cellXStart)*dataOffsetBytes;
FileOffset previousOffset=0;
FileOffset cellOffset;
if (!writer.GetPos(cellOffset)) {
progress.Error("Cannot get cell start position in file");
return false;
}
if (!writer.SetPos(bitmapCellOffset)) {
progress.Error("Cannot go to cell start position in file");
return false;
}
writer.WriteFileOffset(cellOffset-dataStartOffset+1,
dataOffsetBytes);
if (!writer.SetPos(cellOffset)) {
progress.Error("Cannot go back to cell start position in file");
return false;
}
writer.WriteNumber((uint32_t)cell->second.size());
for (std::list<FileOffset>::const_iterator offset=cell->second.begin();
offset!=cell->second.end();
++offset) {
writer.WriteNumber((FileOffset)(*offset-previousOffset));
previousOffset=*offset;
}
}
}
}
return !writer.HasError() && writer.Close();
}
bool Area::ReadOptimized(const TypeConfig& typeConfig,
FileScanner& scanner)
{
if (!scanner.GetPos(fileOffset)) {
return false;
}
TypeId ringType;
bool multipleRings;
uint32_t ringCount=1;
FeatureValueBuffer featureValueBuffer;
scanner.ReadTypeId(ringType,
typeConfig.GetAreaTypeIdBytes());
TypeInfoRef type=typeConfig.GetAreaTypeInfo(ringType);
featureValueBuffer.SetType(type);
if (!featureValueBuffer.Read(scanner,
multipleRings)) {
return false;
}
if (multipleRings) {
if (!scanner.ReadNumber(ringCount)) {
return false;
}
ringCount++;
}
rings.resize(ringCount);
rings[0].featureValueBuffer=featureValueBuffer;
if (ringCount>1) {
rings[0].ring=masterRingId;
}
else {
rings[0].ring=outerRingId;
}
if (!scanner.Read(rings[0].nodes)) {
return false;
}
for (size_t i=1; i<ringCount; i++) {
scanner.ReadTypeId(ringType,
typeConfig.GetAreaTypeIdBytes());
type=typeConfig.GetAreaTypeInfo(ringType);
rings[i].SetType(type);
if (rings[i].featureValueBuffer.GetType()->GetAreaId()!=typeIgnore) {
if (!rings[i].featureValueBuffer.Read(scanner)) {
return false;
}
}
scanner.Read(rings[i].ring);
if (!scanner.Read(rings[i].nodes)) {
return false;
}
}
return !scanner.HasError();
}
