bool Relation::Read(FileScanner& scanner)
{
uint32_t roleCount;
scanner.Read(id);
if (!attributes.Read(scanner)) {
return false;
}
scanner.ReadNumber(roleCount);
if (scanner.HasError()) {
return false;
}
roles.resize(roleCount);
for (size_t i=0; i<roleCount; i++) {
uint32_t nodesCount;
if (!roles[i].attributes.Read(scanner)) {
return false;
}
scanner.Read(roles[i].role);
scanner.ReadNumber(nodesCount);
if (nodesCount>0) {
roles[i].nodes.resize(nodesCount);
Id minId;
uint32_t minLat;
uint32_t minLon;
scanner.Read(minId);
scanner.Read(minLat);
scanner.Read(minLon);
for (size_t j=0; j<nodesCount; j++) {
Id id;
uint32_t latValue;
uint32_t lonValue;
scanner.ReadNumber(id);
scanner.ReadNumber(latValue);
scanner.ReadNumber(lonValue);
roles[i].nodes[j].id=minId+id;
roles[i].nodes[j].lat=(minLat+latValue)/conversionFactor-90.0;
roles[i].nodes[j].lon=(minLon+lonValue)/conversionFactor-180.0;
}
}
}
return !scanner.HasError();
}
bool Way::ReadIds(FileScanner& scanner)
{
ids.resize(nodes.size());
Id minId;
scanner.ReadNumber(minId);
if (minId>0) {
size_t idCurrent=0;
while (idCurrent<ids.size()) {
uint8_t bitset;
size_t bitmask=1;
scanner.Read(bitset);
for (size_t i=0; i<8 && idCurrent<ids.size(); i++) {
if (bitset & bitmask) {
scanner.ReadNumber(ids[idCurrent]);
ids[idCurrent]+=minId;
}
else {
ids[idCurrent]=0;
}
bitmask*=2;
idCurrent++;
}
}
}
return !scanner.HasError();
}
bool Way::ReadOptimized(const TypeConfig& typeConfig,
FileScanner& scanner)
{
if (!scanner.GetPos(fileOffset)) {
return false;
}
TypeId typeId;
scanner.ReadTypeId(typeId,
typeConfig.GetWayTypeIdBytes());
TypeInfoRef type=typeConfig.GetWayTypeInfo(typeId);
featureValueBuffer.SetType(type);
if (!featureValueBuffer.Read(scanner)) {
return false;
}
if (!scanner.Read(nodes)) {
return false;
}
return !scanner.HasError();
}
bool RawNode::Read(const TypeConfig& typeConfig,
FileScanner& scanner)
{
if (!scanner.ReadNumber(id)) {
return false;
}
TypeId typeId;
if (!scanner.ReadTypeId(typeId,
typeConfig.GetNodeTypeIdBytes())) {
return false;
}
TypeInfoRef type=typeConfig.GetNodeTypeInfo(typeId);
featureValueBuffer.SetType(type);
if (!type->GetIgnore()) {
if (!featureValueBuffer.Read(scanner)) {
return false;
}
}
if (!scanner.ReadCoord(coords)) {
return false;
}
return !scanner.HasError();
}
bool LocationIndex::VisitLocationAddresses(const AdminRegion& region,
const Location& location,
AddressVisitor& visitor) const
{
FileScanner scanner;
bool stopped=false;
if (!scanner.Open(AppendFileToDir(path,
FILENAME_LOCATION_IDX),
FileScanner::LowMemRandom,
true)) {
log.Error() << "Cannot open file '" << scanner.GetFilename() << "'!";
return false;
}
if (!scanner.SetPos(indexOffset)) {
return false;
}
if (!VisitLocationAddressEntries(scanner,
region,
location,
visitor,
stopped)) {
return false;
}
return !scanner.HasError() && scanner.Close();
}
bool LocationIndex::ResolveAdminRegionHierachie(const AdminRegionRef& adminRegion,
std::map<FileOffset,AdminRegionRef >& refs) const
{
FileScanner scanner;
if (!scanner.Open(AppendFileToDir(path,
FILENAME_LOCATION_IDX),
FileScanner::LowMemRandom,
true)) {
log.Error() << "Cannot open file '" << scanner.GetFilename() << "'!";
return false;
}
if (!scanner.SetPos(indexOffset)) {
return false;
}
std::list<FileOffset> offsets;
refs[adminRegion->regionOffset]=adminRegion;
if (adminRegion->parentRegionOffset!=0) {
offsets.push_back(adminRegion->parentRegionOffset);
}
while (!offsets.empty()) {
std::list<FileOffset> newOffsets;
for (const auto& offset : offsets) {
if (refs.find(offset)!=refs.end()) {
continue;
}
if (!scanner.SetPos(offset)) {
return false;
}
AdminRegion adminRegion;
if (!LoadAdminRegion(scanner,
adminRegion)) {
return false;
}
refs[adminRegion.regionOffset]=std::make_shared<AdminRegion>(adminRegion);
if (adminRegion.parentRegionOffset!=0) {
newOffsets.push_back(adminRegion.parentRegionOffset);
}
}
offsets.clear();
std::swap(offsets,
newOffsets);
}
return !scanner.HasError() && scanner.Close();
}
bool OptimizeAreasLowZoomGenerator::GetAreas(const TypeConfig& typeConfig,
const ImportParameter& parameter,
Progress& progress,
FileScanner& scanner,
const TypeInfoSet& types,
std::vector<std::list<AreaRef> >& areas,
TypeInfoSet& loadedTypes)
{
uint32_t areaCount=0;
size_t collectedAreasCount=0;
loadedTypes=types;
progress.SetAction("Collecting area data to optimize");
scanner.GotoBegin();
scanner.Read(areaCount);
for (uint32_t a=1; a<=areaCount; a++) {
AreaRef area=std::make_shared<Area>();
progress.SetProgress(a,areaCount);
area->Read(typeConfig,
scanner);
if (loadedTypes.IsSet(area->GetType())) {
areas[area->GetType()->GetIndex()].push_back(area);
collectedAreasCount++;
while (collectedAreasCount>parameter.GetOptimizationMaxWayCount() &&
loadedTypes.Size()>1) {
TypeInfoRef victimType;
for (auto &type : loadedTypes) {
if (areas[type->GetIndex()].size()>0 &&
(!victimType ||
areas[type->GetIndex()].size()<areas[victimType->GetIndex()].size())) {
victimType=type;
}
}
assert(victimType);
collectedAreasCount-=areas[victimType->GetIndex()].size();
areas[victimType->GetIndex()].clear();
loadedTypes.Remove(victimType);
}
}
}
progress.Info("Collected "+NumberToString(collectedAreasCount)+" areas for "+NumberToString(loadedTypes.Size())+" types");
return !scanner.HasError();
}
bool LocationIndex::Load(const std::string& path)
{
this->path=path;
FileScanner scanner;
if (!scanner.Open(AppendFileToDir(path,
FILENAME_LOCATION_IDX),
FileScanner::LowMemRandom,
true)) {
log.Error() << "Cannot open file '" << scanner.GetFilename() << "'!";
return false;
}
if (!(scanner.Read(bytesForNodeFileOffset) &&
scanner.Read(bytesForAreaFileOffset) &&
scanner.Read(bytesForWayFileOffset))) {
return false;
}
uint32_t ignoreTokenCount;
if (!scanner.ReadNumber(ignoreTokenCount)) {
return false;
}
for (size_t i=0; i<ignoreTokenCount; i++) {
std::string token;
if (!scanner.Read(token)) {
return false;
}
regionIgnoreTokens.insert(token);
}
if (!scanner.ReadNumber(ignoreTokenCount)) {
return false;
}
for (size_t i=0; i<ignoreTokenCount; i++) {
std::string token;
if (!scanner.Read(token)) {
return false;
}
locationIgnoreTokens.insert(token);
}
if (!scanner.GetPos(indexOffset)) {
return false;
}
return !scanner.HasError() && scanner.Close();
}
bool LocationIndex::VisitAdminRegions(AdminRegionVisitor& visitor) const
{
FileScanner scanner;
if (!scanner.Open(AppendFileToDir(path,
FILENAME_LOCATION_IDX),
FileScanner::LowMemRandom,
false)) {
log.Error() << "Cannot open file '" << scanner.GetFilename() << "'!";
return false;
}
if (!scanner.SetPos(indexOffset)) {
return false;
}
uint32_t regionCount;
if (!scanner.ReadNumber(regionCount)) {
return false;
}
for (size_t i=0; i<regionCount; i++) {
AdminRegionVisitor::Action action;
FileOffset nextChildOffset;
if (!scanner.ReadFileOffset(nextChildOffset)) {
return false;
}
action=VisitRegionEntries(scanner,
visitor);
if (action==AdminRegionVisitor::error) {
return false;
}
else if (action==AdminRegionVisitor::stop) {
return true;
}
else if (action==AdminRegionVisitor::skipChildren) {
if (i+1<regionCount) {
if (!scanner.SetPos(nextChildOffset)) {
return false;
}
}
}
}
return !scanner.HasError() && scanner.Close();
}
bool LocationIndex::LoadAdminRegion(FileScanner& scanner,
AdminRegion& region) const
{
uint32_t aliasCount;
if (!scanner.GetPos(region.regionOffset)) {
return false;
}
if (!scanner.ReadFileOffset(region.dataOffset)) {
return false;
}
if (!scanner.ReadFileOffset(region.parentRegionOffset)) {
return false;
}
if (!scanner.Read(region.name)) {
return false;
}
if (!Read(scanner,
region.object)) {
return false;
}
if (!scanner.ReadNumber(aliasCount)) {
return false;
}
region.aliases.clear();
if (aliasCount>0) {
region.aliases.resize(aliasCount);
for (size_t i=0; i<aliasCount; i++) {
if (!scanner.Read(region.aliases[i].name)) {
return false;
}
if (!scanner.ReadFileOffset(region.aliases[i].objectOffset,
bytesForNodeFileOffset)) {
return false;
}
}
}
return !scanner.HasError();
}
bool LocationIndex::VisitLocationAddressEntries(FileScanner& scanner,
const AdminRegion& region,
const Location& location,
AddressVisitor& visitor,
bool& stopped) const
{
uint32_t addressCount;
if (!scanner.SetPos(location.addressesOffset)) {
return false;
}
if (!scanner.ReadNumber(addressCount)) {
return false;
}
ObjectFileRefStreamReader objectFileRefReader(scanner);
for (size_t i=0; i<addressCount; i++) {
Address address;
if (!scanner.GetPos(address.addressOffset)) {
return false;
}
address.locationOffset=location.locationOffset;
address.regionOffset=location.regionOffset;
if (!scanner.Read(address.name)) {
return false;
}
if (!objectFileRefReader.Read(address.object)) {
return false;
}
if (!visitor.Visit(region,
location,
address)) {
stopped=true;
break;
}
}
return !scanner.HasError();
}
bool Node::Read(const TypeConfig& typeConfig,
FileScanner& scanner)
{
if (!scanner.GetPos(fileOffset)) {
return false;
}
uint32_t tmpType;
scanner.ReadNumber(tmpType);
TypeInfoRef type=typeConfig.GetTypeInfo((TypeId)tmpType);
featureValueBuffer.SetType(type);
if (!featureValueBuffer.Read(scanner)) {
return false;
}
scanner.ReadCoord(coords);
return !scanner.HasError();
}
bool Database::Open(const std::string& path)
{
assert(!path.empty());
this->path=path;
typeConfig=new TypeConfig();
if (!typeConfig->LoadFromDataFile(path)) {
log.Error() << "Cannot load 'types.dat'!";
return false;
}
FileScanner scanner;
std::string file=AppendFileToDir(path,"bounding.dat");
if (!scanner.Open(file,FileScanner::Normal,true)) {
log.Error() << "Cannot open '" << scanner.GetFilename() << "'";
return false;
}
if (!scanner.ReadBox(boundingBox)) {
log.Error() << "Error while reading '" << scanner.GetFilename() << "'";
}
log.Debug() << "BoundingBox: " << boundingBox.GetDisplayText();
if (scanner.HasError() || !scanner.Close()) {
log.Error() << "Cannot while reading/closing '" << scanner.GetFilename() << "'";
return false;
}
isOpen=true;
return true;
}
bool AreaWayIndexGenerator::CalculateDistribution(const TypeConfigRef& typeConfig,
const ImportParameter& parameter,
Progress& progress,
std::vector<TypeData>& wayTypeData,
size_t& maxLevel) const
{
FileScanner wayScanner;
TypeInfoSet remainingWayTypes;
size_t level;
maxLevel=0;
wayTypeData.resize(typeConfig->GetTypeCount());
if (!wayScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"ways.dat"),
FileScanner::Sequential,
parameter.GetWayDataMemoryMaped())) {
progress.Error("Cannot open 'ways.dat'");
return false;
}
remainingWayTypes.Set(typeConfig->GetWayTypes());
level=parameter.GetAreaWayMinMag();
while (!remainingWayTypes.Empty()) {
uint32_t wayCount=0;
TypeInfoSet currentWayTypes(remainingWayTypes);
double cellWidth=360.0/pow(2.0,(int)level);
double cellHeight=180.0/pow(2.0,(int)level);
std::vector<CoordCountMap> cellFillCount(typeConfig->GetTypeCount());
progress.Info("Scanning Level "+NumberToString(level)+" ("+NumberToString(remainingWayTypes.Size())+" types remaining)");
wayScanner.GotoBegin();
if (!wayScanner.Read(wayCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
Way way;
for (uint32_t w=1; w<=wayCount; w++) {
progress.SetProgress(w,wayCount);
if (!way.Read(*typeConfig,
wayScanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(w)+" of "+
NumberToString(wayCount)+
" in file '"+
wayScanner.GetFilename()+"'");
return false;
}
// Count number of entries per current type and coordinate
if (!currentWayTypes.IsSet(way.GetType())) {
continue;
}
GeoBox boundingBox;
way.GetBoundingBox(boundingBox);
//
// Calculate minimum and maximum tile ids that are covered
// by the way
// Renormalized coordinate space (everything is >=0)
//
uint32_t minxc=(uint32_t)floor((boundingBox.GetMinLon()+180.0)/cellWidth);
uint32_t maxxc=(uint32_t)floor((boundingBox.GetMaxLon()+180.0)/cellWidth);
uint32_t minyc=(uint32_t)floor((boundingBox.GetMinLat()+90.0)/cellHeight);
uint32_t maxyc=(uint32_t)floor((boundingBox.GetMaxLat()+90.0)/cellHeight);
for (uint32_t y=minyc; y<=maxyc; y++) {
for (uint32_t x=minxc; x<=maxxc; x++) {
cellFillCount[way.GetType()->GetIndex()][Pixel(x,y)]++;
}
}
}
// Check if cell fill for current type is in defined limits
for (auto &type : currentWayTypes) {
size_t i=type->GetIndex();
CalculateStatistics(level,
wayTypeData[i],
cellFillCount[i]);
if (!FitsIndexCriteria(parameter,
progress,
*typeConfig->GetTypeInfo(i),
wayTypeData[i],
cellFillCount[i])) {
currentWayTypes.Remove(type);
}
}
for (const auto &type : currentWayTypes) {
maxLevel=std::max(maxLevel,level);
progress.Info("Type "+type->GetName()+", "+NumberToString(wayTypeData[type->GetIndex()].indexCells)+" cells, "+NumberToString(wayTypeData[type->GetIndex()].indexEntries)+" objects");
remainingWayTypes.Remove(type);
}
level++;
}
return !wayScanner.HasError() && wayScanner.Close();
}
AdminRegionVisitor::Action LocationIndex::VisitRegionEntries(FileScanner& scanner,
AdminRegionVisitor& visitor) const
{
AdminRegion region;
uint32_t childCount;
if (!LoadAdminRegion(scanner,
region)) {
return AdminRegionVisitor::error;
}
AdminRegionVisitor::Action action=visitor.Visit(region);
switch (action) {
case AdminRegionVisitor::stop:
return action;
case AdminRegionVisitor::error:
return action;
case AdminRegionVisitor::skipChildren:
return AdminRegionVisitor::skipChildren;
case AdminRegionVisitor::visitChildren:
// just continue...
break;
}
if (!scanner.ReadNumber(childCount)) {
return AdminRegionVisitor::error;
}
for (size_t i=0; i<childCount; i++) {
FileOffset nextChildOffset;
if (!scanner.ReadFileOffset(nextChildOffset)) {
return AdminRegionVisitor::error;
}
action=VisitRegionEntries(scanner,
visitor);
if (action==AdminRegionVisitor::stop ||
action==AdminRegionVisitor::error) {
return action;
}
else if (action==AdminRegionVisitor::skipChildren) {
if (i+1<childCount) {
if (!scanner.SetPos(nextChildOffset)) {
return AdminRegionVisitor::error;
}
}
else {
return AdminRegionVisitor::skipChildren;
}
}
}
if (scanner.HasError()) {
return AdminRegionVisitor::error;
}
else {
return AdminRegionVisitor::visitChildren;
}
}
bool LocationIndex::LoadRegionDataEntry(FileScanner& scanner,
const AdminRegion& adminRegion,
LocationVisitor& visitor,
bool& stopped) const
{
uint32_t poiCount;
uint32_t locationCount;
if (!scanner.ReadNumber(poiCount)) {
return false;
}
ObjectFileRefStreamReader objectFileRefReader(scanner);
for (size_t i=0; i<poiCount; i++) {
POI poi;
poi.regionOffset=adminRegion.regionOffset;
if (!scanner.Read(poi.name)) {
return false;
}
if (!objectFileRefReader.Read(poi.object)) {
return false;
}
if (!visitor.Visit(adminRegion,
poi)) {
stopped=true;
return true;
}
}
if (!scanner.ReadNumber(locationCount)) {
return false;
}
for (size_t i=0; i<locationCount; i++) {
Location location;
uint32_t objectCount;
if (!scanner.GetPos(location.locationOffset)) {
return false;
}
if (!scanner.Read(location.name)) {
return false;
}
location.regionOffset=adminRegion.regionOffset;
if (!scanner.ReadNumber(objectCount)) {
return false;
}
location.objects.reserve(objectCount);
bool hasAddresses;
if (!scanner.Read(hasAddresses)) {
return false;
}
if (hasAddresses) {
if (!scanner.ReadFileOffset(location.addressesOffset)) {
return false;
}
}
else {
location.addressesOffset=0;
}
objectFileRefReader.Reset();
for (size_t j=0; j<objectCount; j++) {
ObjectFileRef ref;
if (!objectFileRefReader.Read(ref)) {
return false;
}
location.objects.push_back(ref);
}
if (!visitor.Visit(adminRegion,
location)) {
stopped=true;
return true;
}
}
return !scanner.HasError();
}
bool RouteNode::Read(FileScanner& scanner)
{
uint32_t objectCount;
uint32_t pathCount;
uint32_t excludesCount;
if (!scanner.GetPos(fileOffset)) {
return false;
}
scanner.ReadNumber(id);
if (!scanner.ReadCoord(coord)) {
return false;
}
scanner.ReadNumber(objectCount);
scanner.ReadNumber(pathCount);
scanner.ReadNumber(excludesCount);
if (scanner.HasError()) {
return false;
}
objects.resize(objectCount);
Id previousFileOffset=0;
for (size_t i=0; i<objectCount; i++) {
RefType type;
FileOffset fileOffset;
if (!scanner.ReadNumber(fileOffset)) {
return false;
}
if (fileOffset % 2==0) {
type=refWay;
}
else {
type=refArea;
}
fileOffset=fileOffset/2;
fileOffset+=previousFileOffset;
objects[i].object.Set(fileOffset,type);
scanner.ReadNumber(objects[i].type);
scanner.Read(objects[i].maxSpeed);
scanner.Read(objects[i].grade);
previousFileOffset=fileOffset;
}
if (pathCount>0) {
GeoCoord minCoord;
paths.resize(pathCount);
for (size_t i=0; i<pathCount; i++) {
uint32_t distanceValue;
scanner.ReadFileOffset(paths[i].offset);
scanner.ReadNumber(paths[i].objectIndex);
//scanner.Read(paths[i].bearing);
scanner.Read(paths[i].flags);
scanner.ReadNumber(distanceValue);
paths[i].distance=distanceValue/(1000.0*100.0);
}
}
excludes.resize(excludesCount);
for (size_t i=0; i<excludesCount; i++) {
scanner.Read(excludes[i].source);
scanner.ReadNumber(excludes[i].targetIndex);
}
return !scanner.HasError();
}
bool Area::ReadOptimized(const TypeConfig& typeConfig,
FileScanner& scanner)
{
if (!scanner.GetPos(fileOffset)) {
return false;
}
TypeId ringType;
bool multipleRings;
uint32_t ringCount=1;
FeatureValueBuffer featureValueBuffer;
scanner.ReadTypeId(ringType,
typeConfig.GetAreaTypeIdBytes());
TypeInfoRef type=typeConfig.GetAreaTypeInfo(ringType);
featureValueBuffer.SetType(type);
if (!featureValueBuffer.Read(scanner,
multipleRings)) {
return false;
}
if (multipleRings) {
if (!scanner.ReadNumber(ringCount)) {
return false;
}
ringCount++;
}
rings.resize(ringCount);
rings[0].featureValueBuffer=featureValueBuffer;
if (ringCount>1) {
rings[0].ring=masterRingId;
}
else {
rings[0].ring=outerRingId;
}
if (!scanner.Read(rings[0].nodes)) {
return false;
}
for (size_t i=1; i<ringCount; i++) {
scanner.ReadTypeId(ringType,
typeConfig.GetAreaTypeIdBytes());
type=typeConfig.GetAreaTypeInfo(ringType);
rings[i].SetType(type);
if (rings[i].featureValueBuffer.GetType()->GetAreaId()!=typeIgnore) {
if (!rings[i].featureValueBuffer.Read(scanner)) {
return false;
}
}
scanner.Read(rings[i].ring);
if (!scanner.Read(rings[i].nodes)) {
return false;
}
}
return !scanner.HasError();
}
bool LocationIndex::VisitRegionLocationEntries(FileScanner& scanner,
LocationVisitor& visitor,
bool recursive,
bool& stopped) const
{
AdminRegion region;
FileOffset childrenOffset;
uint32_t childCount;
if (!LoadAdminRegion(scanner,
region)) {
return false;
}
if (!scanner.GetPos(childrenOffset)) {
return false;
}
if (!scanner.SetPos(region.dataOffset)) {
return false;
}
if (!LoadRegionDataEntry(scanner,
region,
visitor,
stopped)) {
return false;
}
if (stopped || !recursive) {
return !scanner.HasError();
}
if (!scanner.SetPos(childrenOffset)) {
return false;
}
if (!scanner.ReadNumber(childCount)) {
return false;
}
for (size_t i=0; i<childCount; i++) {
FileOffset nextChildOffset;
if (!scanner.ReadFileOffset(nextChildOffset)) {
return false;
}
if (!VisitRegionLocationEntries(scanner,
visitor,
recursive,
stopped)) {
return false;
}
if (stopped) {
break;
}
}
return !scanner.HasError();
}