bool OptimizeAreasLowZoomGenerator::Import(const TypeConfigRef& typeConfig,
const ImportParameter& parameter,
Progress& progress)
{
FileOffset indexOffset=0;
FileWriter writer;
Magnification magnification; // Magnification, we optimize for
TypeInfoSet areaTypes; // Types we optimize
std::list<TypeData> areaTypesData;
GetAreaTypesToOptimize(*typeConfig,
areaTypes);
try {
writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
OptimizeAreasLowZoom::FILE_AREASOPT_DAT));
//
// Write header
//
writer.WriteFileOffset(indexOffset);
if (!HandleAreas(parameter,
progress,
*typeConfig,
writer,
areaTypes,
areaTypesData)) {
progress.Error("Error while optimizing areas");
return false;
}
// Position of the index
indexOffset=writer.GetPos();
if (!WriteHeader(writer,
areaTypesData,
(uint32_t)parameter.GetOptimizationMaxMag())) {
progress.Error("Cannot write file header");
return false;
}
writer.GotoBegin();
writer.WriteFileOffset(indexOffset);
writer.Close();
}
catch (IOException& e) {
progress.Error(e.GetDescription());
writer.CloseFailsafe();
return false;
}
return true;
}
bool Preprocess::ProcessFiles(const TypeConfigRef& typeConfig,
const ImportParameter& parameter,
Progress& progress,
Callback& callback)
{
for (const auto& filename : parameter.GetMapfiles()) {
if (filename.length()>=4 &&
filename.substr(filename.length()-4)==".osm") {
#if defined(HAVE_LIB_XML)
PreprocessOSM preprocess(callback);
if (!preprocess.Import(typeConfig,
parameter,
progress,
filename)) {
return false;
}
#else
progress.Error("Support for the OSM file format is not enabled!");
return false;
#endif
}
else if (filename.length()>=4 &&
filename.substr(filename.length()-4)==".pbf") {
#if defined(HAVE_LIB_PROTOBUF)
PreprocessPBF preprocess(callback);
if (!preprocess.Import(typeConfig,
parameter,
progress,
filename)) {
return false;
}
#else
progress.Error("Support for the PBF file format is not enabled!");
return false;
#endif
}
else {
progress.Error("Sorry, this file type is not yet supported!");
return false;
}
}
return true;
}
bool WayLocationProcessorFilter::AfterProcessingEnd(const ImportParameter& /*parameter*/,
Progress& progress,
const TypeConfig& /*typeConfig*/)
{
delete nameReader;
nameReader=NULL;
delete locationReader;
locationReader=NULL;
writer.SetPos(0);
writer.Write(overallDataCount);
try {
writer.Close();
}
catch (IOException& e) {
progress.Error(e.GetDescription());
writer.CloseFailsafe();
return false;
}
return true;
}
bool PreprocessPBF::ReadBlockHeader(Progress& progress,
FILE* file,
PBF::BlockHeader& blockHeader,
bool silent)
{
uint32_t blockHeaderLength;
if (fread(&blockHeaderLength,4,1,file)!=1) {
if (!silent) {
progress.Error("Cannot read block header length!");
}
return false;
}
uint32_t length=ntohl(blockHeaderLength);
if (length==0 || length>MAX_BLOCK_HEADER_SIZE) {
progress.Error("Block header size invalid!");
return false;
}
AssureBlockSize(length);
if (fread(buffer,sizeof(char),length,file)!=length) {
progress.Error("Cannot read block header!");
delete[] buffer;
return false;
}
if (!blockHeader.ParseFromArray(buffer,length)) {
progress.Error("Cannot parse block header!");
return false;
}
return true;
}
bool Preprocess::Import(const ImportParameter& parameter,
Progress& progress,
const TypeConfig& typeConfig)
{
if (parameter.GetMapfile().length()>=4 &&
parameter.GetMapfile().substr(parameter.GetMapfile().length()-4)==".osm") {
#if defined(HAVE_LIB_XML)
PreprocessOSM preprocess;
return preprocess.Import(parameter,
progress,
typeConfig);
#else
progress.Error("Support for the OSM file format is not enabled!");
#endif
}
if (parameter.GetMapfile().length()>=4 &&
parameter.GetMapfile().substr(parameter.GetMapfile().length()-4)==".pbf") {
#if defined(HAVE_LIB_PROTOBUF)
PreprocessPBF preprocess;
return preprocess.Import(parameter,
progress,
typeConfig);
#else
progress.Error("Support for the PBF file format is not enabled!");
return false;
#endif
}
progress.Error("Sorry, this file type is not yet supported!");
return false;
}
bool WayLocationProcessorFilter::Process(Progress& progress,
const FileOffset& offset,
Way& way,
bool& /*save*/)
{
try {
if (!way.GetType()->GetIndexAsPOI()) {
return true;
}
NameFeatureValue *nameValue=nameReader->GetValue(way.GetFeatureValueBuffer());
if (nameValue==NULL) {
return true;
}
LocationFeatureValue *locationValue=locationReader->GetValue(way.GetFeatureValueBuffer());
std::string name;
std::string location;
std::string address;
name=nameValue->GetName();
if (locationValue!=NULL) {
location=locationValue->GetLocation();
}
writer.WriteFileOffset(offset);
writer.WriteNumber(way.GetType()->GetWayId());
writer.Write(name);
writer.Write(location);
writer.Write(way.nodes);
overallDataCount++;
}
catch (IOException& e) {
progress.Error(e.GetDescription());
return false;
}
return true;
}
static bool ExecuteModules(std::list<ImportModule*>& modules,
const ImportParameter& parameter,
Progress& progress,
const TypeConfigRef& typeConfig)
{
StopClock overAllTimer;
size_t currentStep=1;
for (const auto& module : modules) {
if (currentStep>=parameter.GetStartStep() &&
currentStep<=parameter.GetEndStep()) {
StopClock timer;
bool success;
progress.SetStep(std::string("Step #")+
NumberToString(currentStep)+
" - "+
module->GetDescription());
success=module->Import(typeConfig,
parameter,
progress);
timer.Stop();
progress.Info(std::string("=> ")+timer.ResultString()+" second(s)");
if (!success) {
progress.Error(std::string("Error while executing step '")+module->GetDescription()+"'!");
return false;
}
}
currentStep++;
}
overAllTimer.Stop();
progress.Info(std::string("=> ")+overAllTimer.ResultString()+" second(s)");
return true;
}
bool WayLocationProcessorFilter::BeforeProcessingStart(const ImportParameter& parameter,
Progress& progress,
const TypeConfig& typeConfig)
{
overallDataCount=0;
nameReader=new NameFeatureValueReader(typeConfig);
locationReader=new LocationFeatureValueReader(typeConfig);
try {
writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
SortWayDataGenerator::WAYADDRESS_DAT));
writer.Write(overallDataCount);
}
catch (IOException& e) {
progress.Error(e.GetDescription());
return false;
}
return true;
}
bool OptimizeAreasLowZoomGenerator::HandleAreas(const ImportParameter& parameter,
Progress& progress,
const TypeConfig& typeConfig,
FileWriter& writer,
const TypeInfoSet& types,
std::list<TypeData>& typesData)
{
FileScanner scanner;
// Everything smaller than 2mm should get dropped. Width, height and DPI come from the Nexus 4
double dpi=320.0;
double pixel=2.0/* mm */ * dpi / 25.4 /* inch */;
progress.Info("Minimum visible size in pixel: "+NumberToString((unsigned long)pixel));
try {
scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
AreaDataFile::AREAS_DAT),
FileScanner::Sequential,
parameter.GetWayDataMemoryMaped());
TypeInfoSet typesToProcess(types);
std::vector<std::list<AreaRef> > allAreas(typeConfig.GetTypeCount());
while (true) {
//
// Load type data
//
TypeInfoSet loadedTypes;
if (!GetAreas(typeConfig,
parameter,
progress,
scanner,
typesToProcess,
allAreas,
loadedTypes)) {
return false;
}
typesToProcess.Remove(loadedTypes);
for (const auto& type : loadedTypes) {
progress.SetAction("Optimizing type "+ type->GetName());
for (uint32_t level=parameter.GetOptimizationMinMag();
level<=parameter.GetOptimizationMaxMag();
level++) {
Magnification magnification; // Magnification, we optimize for
std::list<AreaRef> optimizedAreas;
magnification.SetLevel(level);
OptimizeAreas(allAreas[type->GetIndex()],
optimizedAreas,
1280,768,
dpi,
pixel,
magnification,
parameter.GetOptimizationWayMethod());
if (optimizedAreas.empty()) {
progress.Debug("Empty optimization result for level "+NumberToString(level)+", no index generated");
TypeData typeData;
typeData.type=type;
typeData.optLevel=level;
typesData.push_back(typeData);
continue;
}
progress.Info("Optimized from "+NumberToString(allAreas[type->GetIndex()].size())+" to "+NumberToString(optimizedAreas.size())+" areas");
/*
size_t optAreas=optimizedAreas.size();
size_t optRoles=0;
size_t optNodes=0;
for (std::list<AreaRef>::const_iterator a=optimizedAreas.begin();
a!=optimizedAreas.end();
++a) {
AreaRef area=*a;
optRoles+=area->rings.size();
for (size_t r=0; r<area->rings.size(); r++) {
optNodes+=area->rings[r].nodes.size();
}
}*/
/*
std::cout << "Areas: " << origAreas << " => " << optAreas << std::endl;
std::cout << "Roles: " << origRoles << " => " << optRoles << std::endl;
std::cout << "Nodes: " << origNodes << " => " << optNodes << std::endl;*/
TypeData typeData;
typeData.type=type;
typeData.optLevel=level;
GetAreaIndexLevel(parameter,
optimizedAreas,
typeData);
//std::cout << "Resulting index level: " << typeData.indexLevel << ", " << typeData.indexCells << ", " << typeData.indexEntries << std::endl;
FileOffsetFileOffsetMap offsets;
WriteAreas(typeConfig,
writer,
optimizedAreas,
offsets);
if (!WriteAreaBitmap(progress,
writer,
optimizedAreas,
offsets,
typeData)) {
return false;
}
typesData.push_back(typeData);
}
allAreas[type->GetIndex()].clear();
}
if (typesToProcess.Empty()) {
break;
}
}
scanner.Close();
}
catch (IOException& e) {
progress.Error(e.GetDescription());
return false;
}
return true;
}
bool AreaAreaIndexGenerator::Import(const TypeConfigRef& typeConfig,
const ImportParameter& parameter,
Progress& progress)
{
FileScanner scanner;
size_t areas=0; // Number of areas found
size_t areasConsumed=0; // Number of areas consumed
std::vector<double> cellWidth;
std::vector<double> cellHeight;
std::map<Pixel,AreaLeaf> leafs;
std::map<Pixel,AreaLeaf> newAreaLeafs;
cellWidth.resize(parameter.GetAreaAreaIndexMaxMag()+1);
cellHeight.resize(parameter.GetAreaAreaIndexMaxMag()+1);
for (size_t i=0; i<cellWidth.size(); i++) {
cellWidth[i]=360.0/pow(2.0,(int)i);
}
for (size_t i=0; i<cellHeight.size(); i++) {
cellHeight[i]=180.0/pow(2.0,(int)i);
}
//
// Writing index file
//
progress.SetAction("Generating 'areaarea.idx'");
FileWriter writer;
FileOffset topLevelOffset=0;
FileOffset topLevelOffsetOffset; // Offset of the toplevel entry
if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"areaarea.idx"))) {
progress.Error("Cannot create 'areaarea.idx'");
return false;
}
if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"areas.dat"),
FileScanner::Sequential,
parameter.GetWayDataMemoryMaped())) {
progress.Error("Cannot open 'areas.dat'");
return false;
}
writer.WriteNumber((uint32_t)parameter.GetAreaAreaIndexMaxMag()); // MaxMag
if (!writer.GetPos(topLevelOffsetOffset)) {
progress.Error("Cannot read current file position");
return false;
}
if (!writer.WriteFileOffset(topLevelOffset)) {
progress.Error("Cannot write top level entry offset");
return false;
}
int l=parameter.GetAreaAreaIndexMaxMag();
while (l>=0) {
size_t areaLevelEntries=0;
progress.Info(std::string("Storing level ")+NumberToString(l)+"...");
newAreaLeafs.clear();
SetOffsetOfChildren(leafs,newAreaLeafs);
leafs=newAreaLeafs;
// Areas
if (areas==0 ||
(areas>0 && areas>areasConsumed)) {
uint32_t areaCount=0;
progress.Info(std::string("Scanning areas.dat for areas of index level ")+NumberToString(l)+"...");
if (!scanner.GotoBegin()) {
progress.Error("Cannot go to begin of way file");
}
if (!scanner.Read(areaCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
areas=0;
for (uint32_t a=1; a<=areaCount; a++) {
progress.SetProgress(a,areaCount);
FileOffset offset;
Area area;
scanner.GetPos(offset);
if (!area.Read(typeConfig,
scanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(a)+" of "+
NumberToString(areaCount)+
" in file '"+
scanner.GetFilename()+"'");
return false;
}
areas++;
double minLon;
double maxLon;
double minLat;
double maxLat;
area.GetBoundingBox(minLon,maxLon,minLat,maxLat);
//
// Calculate highest level where the bounding box completely
// fits in the cell size and assign area to the tiles that
// hold the geometric center of the tile.
//
int level=parameter.GetAreaAreaIndexMaxMag();
while (level>=0) {
if (maxLon-minLon<=cellWidth[level] &&
maxLat-minLat<=cellHeight[level]) {
break;
}
level--;
}
if (level==l) {
//
// Renormated coordinate space (everything is >=0)
//
minLon+=180;
maxLon+=180;
minLat+=90;
maxLat+=90;
//
// Calculate minimum and maximum tile ids that are covered
// by the area
//
uint32_t minyc=(uint32_t)floor(minLat/cellHeight[level]);
uint32_t maxyc=(uint32_t)ceil(maxLat/cellHeight[level]);
uint32_t minxc=(uint32_t)floor(minLon/cellWidth[level]);
uint32_t maxxc=(uint32_t)ceil(maxLon/cellWidth[level]);
Entry entry;
entry.type=area.GetType()->GetId();
entry.offset=offset;
// Add this area to the tile where the center of the area lies in.
leafs[Pixel((minxc+maxxc)/2,(minyc+maxyc)/2)].areas.push_back(entry);
areaLevelEntries++;
areasConsumed++;
}
}
}
progress.Debug(std::string("Writing ")+NumberToString(leafs.size())+" leafs ("+
NumberToString(areaLevelEntries)+") "+
"to index of level "+NumberToString(l)+"...");
// Remember the offset of one cell in level '0'
if (l==0) {
if (!writer.GetPos(topLevelOffset)) {
progress.Error("Cannot read top level entry offset");
return false;
}
}
/*
uint32_t minX=std::numeric_limits<uint32_t>::max();
uint32_t minY=std::numeric_limits<uint32_t>::max();
uint32_t maxX=std::numeric_limits<uint32_t>::min();
uint32_t maxY=std::numeric_limits<uint32_t>::min();
std::map<TypeId,size_t> useMap;
for (std::map<Pixel,AreaLeaf>::const_iterator leaf=leafs.begin();
leaf!=leafs.end();
++leaf) {
minX=std::min(minX,leaf->first.x);
maxX=std::max(maxX,leaf->first.x);
minY=std::min(minY,leaf->first.y);
maxY=std::max(maxY,leaf->first.y);
for (std::list<Entry>::const_iterator entry=leaf->second.areas.begin();
entry!=leaf->second.areas.end();
entry++) {
std::map<TypeId,size_t>::iterator u=useMap.find(entry->type);
if (u==useMap.end()) {
useMap[entry->type]=1;
}
else {
u->second++;
}
}
}*/
/*
std::cout << "[" << minX << "-" << maxX << "]x[" << minY << "-" << maxY << "] => " << leafs.size() << "/" << (maxX-minX+1)*(maxY-minY+1) << " " << (int)BytesNeededToAddressFileData(leafs.size()) << " " << ByteSizeToString(BytesNeededToAddressFileData(leafs.size())*(maxX-minX+1)*(maxY-minY+1)) << std::endl;
for (std::map<TypeId,size_t>::const_iterator u=useMap.begin();
u!=useMap.end();
++u) {
std::cout << "* " << u->first << " " << typeConfig.GetTypeInfo(u->first).GetName() << " " << u->second << std::endl;
}*/
if (!WriteIndexLevel(parameter,
writer,
(int)l,
leafs)) {
return false;
}
l--;
}
writer.SetPos(topLevelOffsetOffset);
writer.WriteFileOffset(topLevelOffset);
return !writer.HasError() && writer.Close();
}
bool AreaWayIndexGenerator::WriteBitmap(Progress& progress,
FileWriter& writer,
const TypeInfo& typeInfo,
const TypeData& typeData,
const CoordOffsetsMap& typeCellOffsets)
{
size_t indexEntries=0;
size_t dataSize=0;
char buffer[10];
for (CoordOffsetsMap::const_iterator cell=typeCellOffsets.begin();
cell!=typeCellOffsets.end();
++cell) {
indexEntries+=cell->second.size();
dataSize+=EncodeNumber(cell->second.size(),buffer);
FileOffset previousOffset=0;
for (std::list<FileOffset>::const_iterator offset=cell->second.begin();
offset!=cell->second.end();
++offset) {
FileOffset data=*offset-previousOffset;
dataSize+=EncodeNumber(data,buffer);
previousOffset=*offset;
}
}
// "+1" because we add +1 to every offset, to generate offset > 0
uint8_t dataOffsetBytes=BytesNeededToAddressFileData(dataSize+1);
progress.Info("Writing map for "+
typeInfo.GetName()+" , "+
ByteSizeToString(1.0*dataOffsetBytes*typeData.cellXCount*typeData.cellYCount+dataSize));
FileOffset bitmapOffset;
if (!writer.GetPos(bitmapOffset)) {
progress.Error("Cannot get type index start position in file");
return false;
}
assert(typeData.indexOffset!=0);
if (!writer.SetPos(typeData.indexOffset)) {
progress.Error("Cannot go to type index offset in file");
return false;
}
writer.WriteFileOffset(bitmapOffset);
writer.Write(dataOffsetBytes);
if (!writer.SetPos(bitmapOffset)) {
progress.Error("Cannot go to type index start position in file");
return false;
}
// Write the bitmap with offsets for each cell
// We prefill with zero and only overwrite cells that have data
// So zero means "no data for this cell"
for (size_t i=0; i<typeData.cellXCount*typeData.cellYCount; i++) {
writer.WriteFileOffset(0,
dataOffsetBytes);
}
FileOffset dataStartOffset;
if (!writer.GetPos(dataStartOffset)) {
progress.Error("Cannot get start of data section after bitmap");
return false;
}
// Now write the list of offsets of objects for every cell with content
for (CoordOffsetsMap::const_iterator cell=typeCellOffsets.begin();
cell!=typeCellOffsets.end();
++cell) {
FileOffset bitmapCellOffset=bitmapOffset+
((cell->first.y-typeData.cellYStart)*typeData.cellXCount+
cell->first.x-typeData.cellXStart)*(FileOffset)dataOffsetBytes;
FileOffset previousOffset=0;
FileOffset cellOffset;
assert(bitmapCellOffset>=bitmapOffset);
if (!writer.GetPos(cellOffset)) {
progress.Error("Cannot get cell start position in file");
return false;
}
if (!writer.SetPos(bitmapCellOffset)) {
progress.Error("Cannot go to cell start position in file");
return false;
}
assert(cellOffset>bitmapCellOffset);
// We add +1 to make sure, that we can differentiate between "0" as "no entry" and "0" as first data entry.
writer.WriteFileOffset(cellOffset-dataStartOffset+1,dataOffsetBytes);
if (!writer.SetPos(cellOffset)) {
progress.Error("Cannot go back to cell start position in file");
return false;
}
writer.WriteNumber((uint32_t)cell->second.size());
// FileOffsets are already in increasing order, since
// File is scanned from start to end
for (std::list<FileOffset>::const_iterator offset=cell->second.begin();
offset!=cell->second.end();
++offset) {
writer.WriteNumber((FileOffset)(*offset-previousOffset));
previousOffset=*offset;
}
}
return true;
}
bool OptimizeAreaWayIdsGenerator::ScanAreaIds(const ImportParameter& parameter,
Progress& progress,
const TypeConfig& typeConfig,
NodeUseMap& nodeUseMap)
{
FileScanner scanner;
uint32_t dataCount=0;
progress.SetAction("Scanning ids from 'areas2.tmp'");
if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"areas2.tmp"),
FileScanner::Sequential,
parameter.GetAreaDataMemoryMaped())) {
progress.Error(std::string("Cannot open '")+scanner.GetFilename()+"'");
return false;
}
if (!scanner.Read(dataCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
for (uint32_t current=1; current<=dataCount; current++) {
uint8_t type;
Id id;
Area data;
progress.SetProgress(current,dataCount);
if (!scanner.Read(type) ||
!scanner.Read(id) ||
!data.ReadImport(typeConfig,
scanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(current)+" of "+
NumberToString(dataCount)+
" in file '"+
scanner.GetFilename()+"'");
return false;
}
for (const auto& ring: data.rings) {
std::unordered_set<Id> nodeIds;
if (!ring.GetType()->CanRoute()) {
continue;
}
for (const auto id: ring.ids) {
if (nodeIds.find(id)==nodeIds.end()) {
nodeUseMap.SetNodeUsed(id);
nodeIds.insert(id);
}
}
}
}
if (!scanner.Close()) {
progress.Error(std::string("Error while closing file '")+
scanner.GetFilename()+"'");
return false;
}
return true;
}
bool MergeAreasGenerator::Import(const TypeConfigRef& typeConfig,
const ImportParameter& parameter,
Progress& progress)
{
TypeInfoSet mergeTypes;
FileScanner scanner;
FileWriter writer;
uint32_t areasWritten=0;
for (const auto& type : typeConfig->GetTypes()) {
if (type->CanBeArea() &&
type->GetMergeAreas()) {
mergeTypes.Set(type);
}
}
std::unordered_set<Id> nodeUseMap;
try {
scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
MergeAreaDataGenerator::AREAS_TMP),
FileScanner::Sequential,
parameter.GetRawWayDataMemoryMaped());
if (!ScanAreaNodeIds(progress,
*typeConfig,
scanner,
mergeTypes,
nodeUseMap)) {
return false;
}
uint32_t nodeCount=nodeUseMap.size();
progress.Info("Found "+NumberToString(nodeCount)+" nodes as possible connection points for areas");
/* ------ */
writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
AREAS2_TMP));
writer.Write(areasWritten);
while (true) {
TypeInfoSet loadedTypes;
std::vector<AreaMergeData> mergeJob(typeConfig->GetTypeCount());
//
// Load type data
//
progress.SetAction("Collecting area data by type");
if (!GetAreas(parameter,
progress,
*typeConfig,
mergeTypes,
loadedTypes,
nodeUseMap,
scanner,
writer,
mergeJob,
areasWritten)) {
return false;
}
// Merge
progress.SetAction("Merging areas");
for (const auto& type : loadedTypes) {
if (!mergeJob[type->GetIndex()].areas.empty()) {
progress.Info("Merging areas of type "+type->GetName());
MergeAreas(progress,
nodeUseMap,
mergeJob[type->GetIndex()]);
progress.Info("Reduced areas of '"+type->GetName()+"' from "+NumberToString(mergeJob[type->GetIndex()].areaCount)+" to "+NumberToString(mergeJob[type->GetIndex()].areaCount-mergeJob[type->GetIndex()].mergedAway.size()));
mergeJob[type->GetIndex()].areas.clear();
}
}
// Store back merge result
if (!loadedTypes.Empty()) {
if (!WriteMergeResult(progress,
*typeConfig,
scanner,
writer,
loadedTypes,
mergeJob,
areasWritten)) {
return false;
}
mergeTypes.Remove(loadedTypes);
}
if (mergeTypes.Empty()) {
break;
}
}
scanner.Close();
writer.GotoBegin();
writer.Write(areasWritten);
writer.Close();
}
catch (IOException& e) {
progress.Error(e.GetDescription());
scanner.CloseFailsafe();
writer.CloseFailsafe();
return false;
}
return true;
}
bool OptimizeAreaWayIdsGenerator::ScanWayIds(const ImportParameter& parameter,
Progress& progress,
const TypeConfig& typeConfig,
NodeUseMap& nodeUseMap)
{
FileScanner scanner;
uint32_t dataCount=0;
progress.SetAction("Scanning ids from 'wayway.tmp'");
if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"wayway.tmp"),
FileScanner::Sequential,
parameter.GetWayDataMemoryMaped())) {
progress.Error(std::string("Cannot open '")+scanner.GetFilename()+"'");
return false;
}
if (!scanner.Read(dataCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
for (uint32_t current=1; current<=dataCount; current++) {
uint8_t type;
Id id;
Way data;
progress.SetProgress(current,dataCount);
if (!scanner.Read(type) ||
!scanner.Read(id) ||
!data.Read(typeConfig,
scanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(current)+" of "+
NumberToString(dataCount)+
" in file '"+
scanner.GetFilename()+"'");
return false;
}
if (!data.GetType()->CanRoute()) {
continue;
}
std::unordered_set<Id> nodeIds;
for (const auto& id : data.ids) {
if (nodeIds.find(id)==nodeIds.end()) {
nodeUseMap.SetNodeUsed(id);
nodeIds.insert(id);
}
}
// If we have a circular way, we "fake" a double usage,
// to make sure, that the node id of the first node
// is not dropped later on, and we cannot detect
// circular ways anymore
if (data.ids.front()==data.ids.back()) {
nodeUseMap.SetNodeUsed(data.ids.back());
}
}
if (!scanner.Close()) {
progress.Error(std::string("Error while closing file '")+
scanner.GetFilename()+"'");
return false;
}
return true;
}
bool OptimizeAreaWayIdsGenerator::CopyWays(const ImportParameter& parameter,
Progress& progress,
const TypeConfig& typeConfig,
NodeUseMap& nodeUseMap)
{
FileScanner scanner;
FileWriter writer;
uint32_t dataCount=0;
progress.SetAction("Copy data from 'wayway.tmp' to 'ways.tmp'");
if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"wayway.tmp"),
FileScanner::Sequential,
parameter.GetWayDataMemoryMaped())) {
progress.Error(std::string("Cannot open '")+scanner.GetFilename()+"'");
return false;
}
if (!scanner.Read(dataCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"ways.tmp"))) {
progress.Error(std::string("Cannot create '")+writer.GetFilename()+"'");
return false;
}
writer.Write(dataCount);
for (uint32_t current=1; current<=dataCount; current++) {
uint8_t type;
Id id;
Way data;
progress.SetProgress(current,dataCount);
if (!scanner.Read(type) ||
!scanner.Read(id) ||
!data.Read(typeConfig,
scanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(current)+" of "+
NumberToString(dataCount)+
" in file '"+
scanner.GetFilename()+"'");
return false;
}
for (auto& id : data.ids) {
if (!nodeUseMap.IsNodeUsedAtLeastTwice(id)) {
id=0;
}
}
if (!writer.Write(type) ||
!writer.Write(id) ||
!data.Write(typeConfig,
writer)) {
progress.Error(std::string("Error while writing data entry to file '")+
writer.GetFilename()+"'");
return false;
}
}
if (!scanner.Close()) {
progress.Error(std::string("Error while closing file '")+
scanner.GetFilename()+"'");
return false;
}
if (!writer.Close()) {
progress.Error(std::string("Error while closing file '")+
writer.GetFilename()+"'");
return false;
}
return true;
}
bool AreaWayIndexGenerator::Import(const TypeConfigRef& typeConfig,
const ImportParameter& parameter,
Progress& progress)
{
FileScanner wayScanner;
FileWriter writer;
std::vector<TypeData> wayTypeData;
size_t maxLevel;
progress.Info("Minimum magnification: "+NumberToString(parameter.GetAreaWayMinMag()));
//
// Scanning distribution
//
progress.SetAction("Scanning level distribution of way types");
if (!CalculateDistribution(typeConfig,
parameter,
progress,
wayTypeData,
maxLevel)) {
return false;
}
// Calculate number of types which have data
uint32_t indexEntries=0;
for (const auto& type : typeConfig->GetWayTypes())
{
if (wayTypeData[type->GetIndex()].HasEntries()) {
indexEntries++;
}
}
//
// Writing index file
//
progress.SetAction("Generating 'areaway.idx'");
if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"areaway.idx"))) {
progress.Error("Cannot create 'areaway.idx'");
return false;
}
writer.Write(indexEntries);
for (const auto &type : typeConfig->GetWayTypes()) {
size_t i=type->GetIndex();
if (wayTypeData[i].HasEntries()) {
uint8_t dataOffsetBytes=0;
FileOffset bitmapOffset=0;
writer.WriteTypeId(type->GetWayId(),
typeConfig->GetWayTypeIdBytes());
writer.GetPos(wayTypeData[i].indexOffset);
writer.WriteFileOffset(bitmapOffset);
writer.Write(dataOffsetBytes);
writer.WriteNumber(wayTypeData[i].indexLevel);
writer.WriteNumber(wayTypeData[i].cellXStart);
writer.WriteNumber(wayTypeData[i].cellXEnd);
writer.WriteNumber(wayTypeData[i].cellYStart);
writer.WriteNumber(wayTypeData[i].cellYEnd);
}
}
if (!wayScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"ways.dat"),
FileScanner::Sequential,
parameter.GetWayDataMemoryMaped())) {
progress.Error("Cannot open 'ways.dat'");
return false;
}
for (size_t l=parameter.GetAreaWayMinMag(); l<=maxLevel; l++) {
TypeInfoSet indexTypes(*typeConfig);
uint32_t wayCount;
double cellWidth=360.0/pow(2.0,(int)l);
double cellHeight=180.0/pow(2.0,(int)l);
wayScanner.GotoBegin();
for (const auto &type : typeConfig->GetWayTypes()) {
if (wayTypeData[type->GetIndex()].HasEntries() &&
wayTypeData[type->GetIndex()].indexLevel==l) {
indexTypes.Set(type);
}
}
if (indexTypes.Empty()) {
continue;
}
progress.Info("Scanning ways for index level "+NumberToString(l));
std::vector<CoordOffsetsMap> typeCellOffsets(typeConfig->GetTypeCount());
if (!wayScanner.Read(wayCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
Way way;
for (uint32_t w=1; w<=wayCount; w++) {
progress.SetProgress(w,wayCount);
FileOffset offset;
wayScanner.GetPos(offset);
if (!way.Read(*typeConfig,
wayScanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(w)+" of "+
NumberToString(wayCount)+
" in file '"+
wayScanner.GetFilename()+"'");
return false;
}
if (!indexTypes.IsSet(way.GetType())) {
continue;
}
GeoBox boundingBox;
way.GetBoundingBox(boundingBox);
//
// Calculate minimum and maximum tile ids that are covered
// by the way
// Renormalized coordinate space (everything is >=0)
//
uint32_t minxc=(uint32_t)floor((boundingBox.GetMinLon()+180.0)/cellWidth);
uint32_t maxxc=(uint32_t)floor((boundingBox.GetMaxLon()+180.0)/cellWidth);
uint32_t minyc=(uint32_t)floor((boundingBox.GetMinLat()+90.0)/cellHeight);
uint32_t maxyc=(uint32_t)floor((boundingBox.GetMaxLat()+90.0)/cellHeight);
for (uint32_t y=minyc; y<=maxyc; y++) {
for (uint32_t x=minxc; x<=maxxc; x++) {
typeCellOffsets[way.GetType()->GetIndex()][Pixel(x,y)].push_back(offset);
}
}
}
for (const auto &type : indexTypes) {
size_t index=type->GetIndex();
if (!WriteBitmap(progress,
writer,
*typeConfig->GetTypeInfo(index),
wayTypeData[index],
typeCellOffsets[index])) {
return false;
}
}
}
return !writer.HasError() && writer.Close();
}
bool AreaWayIndexGenerator::Import(const ImportParameter& parameter,
Progress& progress,
const TypeConfig& typeConfig)
{
FileScanner wayScanner;
FileWriter writer;
std::set<TypeId> remainingWayTypes;
std::vector<TypeData> wayTypeData;
size_t level;
size_t maxLevel=0;
wayTypeData.resize(typeConfig.GetTypes().size());
if (!wayScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"ways.dat"),
FileScanner::Sequential,
parameter.GetWayDataMemoryMaped())) {
progress.Error("Cannot open 'ways.dat'");
return false;
}
//
// Scanning distribution
//
progress.SetAction("Scanning level distribution of way types");
for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
if (typeConfig.GetTypeInfo(i).CanBeWay() &&
!typeConfig.GetTypeInfo(i).GetIgnore()) {
remainingWayTypes.insert(i);
}
}
level=parameter.GetAreaWayMinMag();
while (!remainingWayTypes.empty()) {
uint32_t wayCount=0;
std::set<TypeId> currentWayTypes(remainingWayTypes);
double cellWidth=360.0/pow(2.0,(int)level);
double cellHeight=180.0/pow(2.0,(int)level);
std::vector<CoordCountMap> cellFillCount(typeConfig.GetTypes().size());
progress.Info("Scanning Level "+NumberToString(level)+" ("+NumberToString(remainingWayTypes.size())+" types remaining)");
wayScanner.GotoBegin();
if (!wayScanner.Read(wayCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
Way way;
for (uint32_t w=1; w<=wayCount; w++) {
progress.SetProgress(w,wayCount);
if (!way.Read(wayScanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(w)+" of "+
NumberToString(wayCount)+
" in file '"+
wayScanner.GetFilename()+"'");
return false;
}
// Count number of entries per current type and coordinate
if (currentWayTypes.find(way.GetType())==currentWayTypes.end()) {
continue;
}
double minLon;
double maxLon;
double minLat;
double maxLat;
way.GetBoundingBox(minLon,maxLon,minLat,maxLat);
//
// Calculate minimum and maximum tile ids that are covered
// by the way
// Renormated coordinate space (everything is >=0)
//
uint32_t minxc=(uint32_t)floor((minLon+180.0)/cellWidth);
uint32_t maxxc=(uint32_t)floor((maxLon+180.0)/cellWidth);
uint32_t minyc=(uint32_t)floor((minLat+90.0)/cellHeight);
uint32_t maxyc=(uint32_t)floor((maxLat+90.0)/cellHeight);
for (uint32_t y=minyc; y<=maxyc; y++) {
for (uint32_t x=minxc; x<=maxxc; x++) {
cellFillCount[way.GetType()][Pixel(x,y)]++;
}
}
}
// Check if cell fill for current type is in defined limits
for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
if (currentWayTypes.find(i)!=currentWayTypes.end()) {
CalculateStatistics(level,wayTypeData[i],cellFillCount[i]);
if (!FitsIndexCriteria(parameter,
progress,
typeConfig.GetTypeInfo(i),
wayTypeData[i],
cellFillCount[i])) {
currentWayTypes.erase(i);
}
}
}
for (std::set<TypeId>::const_iterator cwt=currentWayTypes.begin();
cwt!=currentWayTypes.end();
cwt++) {
maxLevel=std::max(maxLevel,level);
progress.Info("Type "+typeConfig.GetTypeInfo(*cwt).GetName()+"(" + NumberToString(*cwt)+"), "+NumberToString(wayTypeData[*cwt].indexCells)+" cells, "+NumberToString(wayTypeData[*cwt].indexEntries)+" objects");
remainingWayTypes.erase(*cwt);
}
level++;
}
//
// Writing index file
//
progress.SetAction("Generating 'areaway.idx'");
if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"areaway.idx"))) {
progress.Error("Cannot create 'areaway.idx'");
return false;
}
uint32_t indexEntries=0;
for (size_t i=0; i<typeConfig.GetTypes().size(); i++)
{
if (typeConfig.GetTypeInfo(i).CanBeWay() &&
wayTypeData[i].HasEntries()) {
indexEntries++;
}
}
writer.Write(indexEntries);
for (size_t i=0; i<typeConfig.GetTypes().size(); i++)
{
if (typeConfig.GetTypeInfo(i).CanBeWay() &&
wayTypeData[i].HasEntries()) {
uint8_t dataOffsetBytes=0;
FileOffset bitmapOffset=0;
writer.WriteNumber(typeConfig.GetTypeInfo(i).GetId());
writer.GetPos(wayTypeData[i].indexOffset);
writer.WriteFileOffset(bitmapOffset);
if (wayTypeData[i].HasEntries()) {
writer.Write(dataOffsetBytes);
writer.WriteNumber(wayTypeData[i].indexLevel);
writer.WriteNumber(wayTypeData[i].cellXStart);
writer.WriteNumber(wayTypeData[i].cellXEnd);
writer.WriteNumber(wayTypeData[i].cellYStart);
writer.WriteNumber(wayTypeData[i].cellYEnd);
}
}
}
for (size_t l=parameter.GetAreaWayMinMag(); l<=maxLevel; l++) {
std::set<TypeId> indexTypes;
uint32_t wayCount;
double cellWidth=360.0/pow(2.0,(int)l);
double cellHeight=180.0/pow(2.0,(int)l);
for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
if (typeConfig.GetTypeInfo(i).CanBeWay() &&
wayTypeData[i].HasEntries() &&
wayTypeData[i].indexLevel==l) {
indexTypes.insert(i);
}
}
if (indexTypes.empty()) {
continue;
}
progress.Info("Scanning ways for index level "+NumberToString(l));
std::vector<CoordOffsetsMap> typeCellOffsets(typeConfig.GetTypes().size());
wayScanner.GotoBegin();
if (!wayScanner.Read(wayCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
Way way;
for (uint32_t w=1; w<=wayCount; w++) {
progress.SetProgress(w,wayCount);
FileOffset offset;
wayScanner.GetPos(offset);
if (!way.Read(wayScanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(w)+" of "+
NumberToString(wayCount)+
" in file '"+
wayScanner.GetFilename()+"'");
return false;
}
if (indexTypes.find(way.GetType())==indexTypes.end()) {
continue;
}
double minLon;
double maxLon;
double minLat;
double maxLat;
way.GetBoundingBox(minLon,maxLon,minLat,maxLat);
//
// Calculate minimum and maximum tile ids that are covered
// by the way
// Renormated coordinate space (everything is >=0)
//
uint32_t minxc=(uint32_t)floor((minLon+180.0)/cellWidth);
uint32_t maxxc=(uint32_t)floor((maxLon+180.0)/cellWidth);
uint32_t minyc=(uint32_t)floor((minLat+90.0)/cellHeight);
uint32_t maxyc=(uint32_t)floor((maxLat+90.0)/cellHeight);
for (uint32_t y=minyc; y<=maxyc; y++) {
for (uint32_t x=minxc; x<=maxxc; x++) {
typeCellOffsets[way.GetType()][Pixel(x,y)].push_back(offset);
}
}
}
for (std::set<TypeId>::const_iterator type=indexTypes.begin();
type!=indexTypes.end();
++type) {
if (!WriteBitmap(progress,
writer,
typeConfig.GetTypeInfo(*type),
wayTypeData[*type],
typeCellOffsets[*type])) {
return false;
}
}
}
return !writer.HasError() && writer.Close();
}
bool PreprocessPBF::ReadPrimitiveBlock(Progress& progress,
FILE* file,
const PBF::BlockHeader& blockHeader,
PBF::PrimitiveBlock& primitiveBlock)
{
PBF::Blob blob;
google::protobuf::int32 length=blockHeader.datasize();
if (length==0 || length>MAX_BLOB_SIZE) {
progress.Error("Blob size invalid!");
return false;
}
AssureBlockSize(length);
if (fread(buffer,sizeof(char),length,file)!=(size_t)length) {
progress.Error("Cannot read blob!");
return false;
}
if (!blob.ParseFromArray(buffer,length)) {
progress.Error("Cannot parse blob!");
return false;
}
if (blob.has_raw()) {
length=(uint32_t)blob.raw().length();
AssureBlockSize(length);
memcpy(buffer,blob.raw().data(),(size_t)length);
}
else if (blob.has_zlib_data()) {
#if defined(HAVE_LIB_ZLIB)
length=blob.raw_size();
AssureBlockSize(length);
z_stream compressedStream;
compressedStream.next_in=(Bytef*)const_cast<char*>(blob.zlib_data().data());
compressedStream.avail_in=(uint32_t)blob.zlib_data().size();
compressedStream.next_out=(Bytef*)buffer;
compressedStream.avail_out=(uInt)length;
compressedStream.zalloc=Z_NULL;
compressedStream.zfree=Z_NULL;
compressedStream.opaque=Z_NULL;
if (inflateInit( &compressedStream)!=Z_OK) {
progress.Error("Cannot decode zlib compressed blob data!");
return false;
}
if (inflate(&compressedStream,Z_FINISH)!=Z_STREAM_END) {
progress.Error("Cannot decode zlib compressed blob data!");
return false;
}
if (inflateEnd(&compressedStream)!=Z_OK) {
progress.Error("Cannot decode zlib compressed blob data!");
return false;
}
#else
progress.Error("Data is zlib encoded but zlib support is not enabled!");
return false;
#endif
}
else if (blob.has_bzip2_data()) {
progress.Error("Data is bzip2 encoded but bzip2 support is not enabled!");
return false;
}
else if (blob.has_lzma_data()) {
progress.Error("Data is lzma encoded but lzma support is not enabled!");
return false;
}
if (!primitiveBlock.ParseFromArray(buffer,length)) {
progress.Error("Cannot parse primitive block!");
return false;
}
return true;
}
bool PreprocessPBF::Import(const TypeConfigRef& typeConfig,
const ImportParameter& /*parameter*/,
Progress& progress,
const std::string& filename)
{
FileOffset fileSize;
FileOffset currentPosition;
progress.SetAction(std::string("Parsing *.osm.pbf file '")+filename+"'");
try {
fileSize=GetFileSize(filename);
FILE* file;
file=fopen(filename.c_str(),"rb");
if (file==NULL) {
progress.Error("Cannot open file!");
return false;
}
// BlockHeader
PBF::BlockHeader blockHeader;
if (!ReadBlockHeader(progress,file,blockHeader,false)) {
fclose(file);
return false;
}
if (blockHeader.type()!="OSMHeader") {
progress.Error("File '"+filename+"' is not an OSM PBF file!");
fclose(file);
return false;
}
PBF::HeaderBlock headerBlock;
if (!ReadHeaderBlock(progress,
file,
blockHeader,
headerBlock)) {
fclose(file);
return false;
}
for (int i=0; i<headerBlock.required_features_size(); i++) {
std::string feature=headerBlock.required_features(i);
if (feature!="OsmSchema-V0.6" &&
feature!="DenseNodes") {
progress.Error(std::string("Unsupported feature '")+feature+"'");
fclose(file);
return false;
}
}
nodes.reserve(20000);
members.reserve(2000);
std::future<void> currentBlockTask;
while (true) {
PBF::BlockHeader blockHeader;
if (!GetPos(file,
currentPosition)) {
progress.Error("Cannot read current position in '"+filename+"'!");
fclose(file);
return false;
}
progress.SetProgress(currentPosition,
fileSize);
if (!ReadBlockHeader(progress,
file,
blockHeader,
true)) {
fclose(file);
break;
}
if (blockHeader.type()!="OSMData") {
progress.Error("File '"+filename+"' is not an OSM PBF file!");
fclose(file);
return false;
}
std::unique_ptr<PBF::PrimitiveBlock> block(new PBF::PrimitiveBlock());
if (!ReadPrimitiveBlock(progress,
file,
blockHeader,
*block)) {
fclose(file);
return false;
}
if (currentBlockTask.valid()) {
currentBlockTask.get();
}
currentBlockTask=std::async(std::launch::async,
&PreprocessPBF::ProcessBlock,this,
typeConfig,
std::move(block));
}
if (currentBlockTask.valid()) {
currentBlockTask.get();
}
}
catch (IOException& e) {
progress.Error(e.GetDescription());
return false;
}
return true;
}
bool Preprocess::Cleanup(Progress& progress)
{
if (currentPageId!=0) {
StoreCurrentPage();
}
nodeWriter.SetPos(0);
nodeWriter.Write(nodeCount);
wayWriter.SetPos(0);
wayWriter.Write(wayCount+areaCount);
relationWriter.SetPos(0);
relationWriter.Write(relationCount);
coastlineWriter.SetPos(0);
coastlineWriter.Write(coastlineCount);
coordWriter.SetPos(0);
coordWriter.Write(coordPageSize);
FileOffset coordIndexOffset=coordPageCount*coordPageSize*2*sizeof(uint32_t);
coordWriter.Write(coordIndexOffset);
coordWriter.SetPos(coordIndexOffset);
coordWriter.Write((uint32_t)coordIndex.size());
for (CoordPageOffsetMap::const_iterator entry=coordIndex.begin();
entry!=coordIndex.end();
++entry) {
coordWriter.Write(entry->first);
coordWriter.Write(entry->second);
}
nodeWriter.Close();
wayWriter.Close();
relationWriter.Close();
coastlineWriter.Close();
coordWriter.Close();
progress.Info(std::string("Nodes: ")+NumberToString(nodeCount));
progress.Info(std::string("Ways/Areas/Sum: ")+NumberToString(wayCount)+" "+
NumberToString(areaCount)+" "+
NumberToString(wayCount+areaCount));
progress.Info(std::string("Relations: ")+NumberToString(relationCount));
progress.Info(std::string("Coastlines: ")+NumberToString(coastlineCount));
progress.Info(std::string("Coord pages: ")+NumberToString(coordIndex.size()));
if (nodeSortingError) {
progress.Error("Nodes are not sorted by increasing id");
}
if (waySortingError) {
progress.Error("Ways are not sorted by increasing id");
}
if (relationSortingError) {
progress.Error("Relations are not sorted by increasing id");
}
if (nodeSortingError || waySortingError || relationSortingError) {
return false;
}
return true;
}
bool NodeDataGenerator::Import(const TypeConfigRef& typeConfig,
const ImportParameter& parameter,
Progress& progress)
{
uint32_t rawNodeCount=0;
uint32_t nodesReadCount=0;
uint32_t nodesWrittenCount=0;
//
// Iterator over all raw nodes, hcekc they type, and convert them from raw nodes
// to nodes if the type is interesting (!=typeIgnore).
//
// Count the bounding box by the way...
//
progress.SetAction("Generating nodes.tmp");
FileScanner scanner;
FileWriter writer;
if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"rawnodes.dat"),
FileScanner::Sequential,
parameter.GetRawNodeDataMemoryMaped())) {
progress.Error("Cannot open 'rawnodes.dat'");
return false;
}
if (!scanner.Read(rawNodeCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"nodes.tmp"))) {
progress.Error("Cannot create 'nodes.tmp'");
return false;
}
writer.Write(nodesWrittenCount);
for (uint32_t n=1; n<=rawNodeCount; n++) {
progress.SetProgress(n,rawNodeCount);
RawNode rawNode;
Node node;
if (!rawNode.Read(typeConfig,
scanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(n)+" of "+
NumberToString(rawNodeCount)+
" in file '"+
scanner.GetFilename()+"'");
return false;
}
nodesReadCount++;
if (!rawNode.GetType()->GetIgnore()) {
node.SetFeatures(rawNode.GetFeatureValueBuffer());
node.SetCoords(rawNode.GetCoords());
FileOffset fileOffset;
if (!writer.GetPos(fileOffset)) {
progress.Error(std::string("Error while reading current fileOffset in file '")+
writer.GetFilename()+"'");
return false;
}
writer.Write(rawNode.GetId());
node.Write(typeConfig,
writer);
nodesWrittenCount++;
}
}
if (!scanner.Close()) {
return false;
}
writer.SetPos(0);
writer.Write(nodesWrittenCount);
if (!writer.Close()) {
return false;
}
progress.Info(std::string("Read "+NumberToString(nodesReadCount)+" nodes, wrote "+NumberToString(nodesWrittenCount)+" nodes"));
return true;
}
bool AreaNodeIndexGenerator::Import(const ImportParameter& parameter,
Progress& progress,
const TypeConfig& typeConfig)
{
FileScanner nodeScanner;
FileWriter writer;
std::set<TypeId> remainingNodeTypes; //! Set of types we still must process
std::vector<TypeData> nodeTypeData;
size_t level;
size_t maxLevel=0;
nodeTypeData.resize(typeConfig.GetTypes().size());
if (!nodeScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"nodes.dat"),
FileScanner::Sequential,
true)) {
progress.Error("Cannot open 'nodes.dat'");
return false;
}
//
// Scanning distribution
//
progress.SetAction("Scanning level distribution of node types");
// Initially we must process all types that represents nodes and that should
// not be ignored
for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
if (typeConfig.GetTypeInfo(i).CanBeNode() &&
!typeConfig.GetTypeInfo(i).GetIgnore()) {
remainingNodeTypes.insert(i);
}
}
level=parameter.GetAreaNodeMinMag();
while (!remainingNodeTypes.empty()) {
uint32_t nodeCount=0;
std::set<TypeId> currentNodeTypes(remainingNodeTypes);
double cellWidth=360.0/pow(2.0,(int)level);
double cellHeight=180.0/pow(2.0,(int)level);
std::vector<std::map<Pixel,size_t> > cellFillCount;
cellFillCount.resize(typeConfig.GetTypes().size());
progress.Info("Scanning Level "+NumberToString(level)+" ("+NumberToString(remainingNodeTypes.size())+" types still to process)");
nodeScanner.GotoBegin();
if (!nodeScanner.Read(nodeCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
for (uint32_t n=1; n<=nodeCount; n++) {
progress.SetProgress(n,nodeCount);
FileOffset offset;
Node node;
nodeScanner.GetPos(offset);
if (!node.Read(nodeScanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(n)+" of "+
NumberToString(nodeCount)+
" in file '"+
nodeScanner.GetFilename()+"'");
return false;
}
// If we still need to handle this type,
// count number of entries per type and tile cell
if (currentNodeTypes.find(node.GetType())!=currentNodeTypes.end()) {
uint32_t xc=(uint32_t)floor((node.GetLon()+180.0)/cellWidth);
uint32_t yc=(uint32_t)floor((node.GetLat()+90.0)/cellHeight);
cellFillCount[node.GetType()][Pixel(xc,yc)]++;
}
}
// Check statistics for each type
// If statistics are within goal limits, use this level
// for this type (else try again with the next higher level)
for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
if (currentNodeTypes.find(i)!=currentNodeTypes.end()) {
size_t entryCount=0;
size_t max=0;
nodeTypeData[i].indexLevel=(uint32_t)level;
nodeTypeData[i].indexCells=cellFillCount[i].size();
nodeTypeData[i].indexEntries=0;
if (!cellFillCount[i].empty()) {
nodeTypeData[i].cellXStart=cellFillCount[i].begin()->first.x;
nodeTypeData[i].cellYStart=cellFillCount[i].begin()->first.y;
nodeTypeData[i].cellXEnd=nodeTypeData[i].cellXStart;
nodeTypeData[i].cellYEnd=nodeTypeData[i].cellYStart;
for (std::map<Pixel,size_t>::const_iterator cell=cellFillCount[i].begin();
cell!=cellFillCount[i].end();
++cell) {
nodeTypeData[i].indexEntries+=cell->second;
nodeTypeData[i].cellXStart=std::min(nodeTypeData[i].cellXStart,cell->first.x);
nodeTypeData[i].cellXEnd=std::max(nodeTypeData[i].cellXEnd,cell->first.x);
nodeTypeData[i].cellYStart=std::min(nodeTypeData[i].cellYStart,cell->first.y);
nodeTypeData[i].cellYEnd=std::max(nodeTypeData[i].cellYEnd,cell->first.y);
}
}
nodeTypeData[i].cellXCount=nodeTypeData[i].cellXEnd-nodeTypeData[i].cellXStart+1;
nodeTypeData[i].cellYCount=nodeTypeData[i].cellYEnd-nodeTypeData[i].cellYStart+1;
// Count absolute number of entries
for (std::map<Pixel,size_t>::const_iterator cell=cellFillCount[i].begin();
cell!=cellFillCount[i].end();
++cell) {
entryCount+=cell->second;
max=std::max(max,cell->second);
}
// Average number of entries per tile cell
double average=entryCount*1.0/cellFillCount[i].size();
// If we do not have any entries, we store it now
if (cellFillCount[i].empty()) {
continue;
}
// If the fill rate of the index is too low, we use this index level anyway
if (nodeTypeData[i].indexCells/(1.0*nodeTypeData[i].cellXCount*nodeTypeData[i].cellYCount)<=
parameter.GetAreaNodeIndexMinFillRate()) {
progress.Warning(typeConfig.GetTypeInfo(i).GetName()+" ("+NumberToString(i)+") is not well distributed");
continue;
}
// If average fill size and max fill size for tile cells
// is within limits, store it now.
if (max<=parameter.GetAreaNodeIndexCellSizeMax() &&
average<=parameter.GetAreaNodeIndexCellSizeAverage()) {
continue;
}
// else, we remove it from the list and try again with an higher
// level.
currentNodeTypes.erase(i);
}
}
// Now process all types for this limit, that are within the limits
for (std::set<TypeId>::const_iterator cnt=currentNodeTypes.begin();
cnt!=currentNodeTypes.end();
cnt++) {
maxLevel=std::max(maxLevel,level);
progress.Info("Type "+typeConfig.GetTypeInfo(*cnt).GetName()+"(" + NumberToString(*cnt)+"), "+NumberToString(nodeTypeData[*cnt].indexCells)+" cells, "+NumberToString(nodeTypeData[*cnt].indexEntries)+" objects");
remainingNodeTypes.erase(*cnt);
}
level++;
}
//
// Writing index file
//
progress.SetAction("Generating 'areanode.idx'");
if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"areanode.idx"))) {
progress.Error("Cannot create 'areanode.idx'");
return false;
}
uint32_t indexEntries=0;
// Count number of types in index
for (size_t i=0; i<typeConfig.GetTypes().size(); i++)
{
if (typeConfig.GetTypeInfo(i).CanBeNode() &&
nodeTypeData[i].HasEntries()) {
indexEntries++;
}
}
writer.Write(indexEntries);
// Store index data for each type
for (size_t i=0; i<typeConfig.GetTypes().size(); i++)
{
if (typeConfig.GetTypeInfo(i).CanBeNode() &&
nodeTypeData[i].HasEntries()) {
FileOffset bitmapOffset=0;
uint8_t dataOffsetBytes=0;
writer.WriteNumber(typeConfig.GetTypeInfo(i).GetId());
writer.GetPos(nodeTypeData[i].indexOffset);
writer.WriteFileOffset(bitmapOffset);
writer.Write(dataOffsetBytes);
writer.WriteNumber(nodeTypeData[i].indexLevel);
writer.WriteNumber(nodeTypeData[i].cellXStart);
writer.WriteNumber(nodeTypeData[i].cellXEnd);
writer.WriteNumber(nodeTypeData[i].cellYStart);
writer.WriteNumber(nodeTypeData[i].cellYEnd);
}
}
// Now store index bitmap for each type in increasing level order (why?)
for (size_t l=0; l<=maxLevel; l++) {
std::set<TypeId> indexTypes;
uint32_t nodeCount;
double cellWidth=360.0/pow(2.0,(int)l);
double cellHeight=180.0/pow(2.0,(int)l);
for (size_t i=0; i<typeConfig.GetTypes().size(); i++) {
if (typeConfig.GetTypeInfo(i).CanBeNode() &&
nodeTypeData[i].HasEntries() &&
nodeTypeData[i].indexLevel==l) {
indexTypes.insert(i);
}
}
if (indexTypes.empty()) {
continue;
}
progress.Info("Scanning nodes for index level "+NumberToString(l));
std::vector<std::map<Pixel,std::list<FileOffset> > > typeCellOffsets;
typeCellOffsets.resize(typeConfig.GetTypes().size());
nodeScanner.GotoBegin();
if (!nodeScanner.Read(nodeCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
//
// Collect all offsets
//
for (uint32_t n=1; n<=nodeCount; n++) {
progress.SetProgress(n,nodeCount);
FileOffset offset;
Node node;
nodeScanner.GetPos(offset);
if (!node.Read(nodeScanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(n)+" of "+
NumberToString(nodeCount)+
" in file '"+
nodeScanner.GetFilename()+"'");
return false;
}
if (indexTypes.find(node.GetType())!=indexTypes.end()) {
uint32_t xc=(uint32_t)floor((node.GetLon()+180.0)/cellWidth);
uint32_t yc=(uint32_t)floor((node.GetLat()+90.0)/cellHeight);
typeCellOffsets[node.GetType()][Pixel(xc,yc)].push_back(offset);
}
}
//
// Write bitmap
//
for (std::set<TypeId>::const_iterator type=indexTypes.begin();
type!=indexTypes.end();
++type) {
size_t indexEntries=0;
size_t dataSize=0;
char buffer[10];
for (std::map<Pixel,std::list<FileOffset> >::const_iterator cell=typeCellOffsets[*type].begin();
cell!=typeCellOffsets[*type].end();
++cell) {
indexEntries+=cell->second.size();
dataSize+=EncodeNumber(cell->second.size(),buffer);
FileOffset previousOffset=0;
for (std::list<FileOffset>::const_iterator offset=cell->second.begin();
offset!=cell->second.end();
++offset) {
FileOffset data=*offset-previousOffset;
dataSize+=EncodeNumber(data,buffer);
previousOffset=*offset;
}
}
// "+1" because we add +1 to every offset, to generate offset > 0
uint8_t dataOffsetBytes=BytesNeededToAddressFileData(dataSize);
progress.Info("Writing map for "+
typeConfig.GetTypeInfo(*type).GetName()+", "+
NumberToString(typeCellOffsets[*type].size())+" cells, "+
NumberToString(indexEntries)+" entries, "+
ByteSizeToString(1.0*dataOffsetBytes*nodeTypeData[*type].cellXCount*nodeTypeData[*type].cellYCount));
FileOffset bitmapOffset;
if (!writer.GetPos(bitmapOffset)) {
progress.Error("Cannot get type index start position in file");
return false;
}
assert(nodeTypeData[*type].indexOffset!=0);
if (!writer.SetPos(nodeTypeData[*type].indexOffset)) {
progress.Error("Cannot go to type index offset in file");
return false;
}
writer.WriteFileOffset(bitmapOffset);
writer.Write(dataOffsetBytes);
if (!writer.SetPos(bitmapOffset)) {
progress.Error("Cannot go to type index start position in file");
return false;
}
// Write the bitmap with offsets for each cell
// We prefill with zero and only overwrite cells that have data
// So zero means "no data for this cell"
for (size_t i=0; i<nodeTypeData[*type].cellXCount*nodeTypeData[*type].cellYCount; i++) {
FileOffset cellOffset=0;
writer.WriteFileOffset(cellOffset,
dataOffsetBytes);
}
FileOffset dataStartOffset;
if (!writer.GetPos(dataStartOffset)) {
progress.Error("Cannot get start of data section after bitmap");
return false;
}
// Now write the list of offsets of objects for every cell with content
for (std::map<Pixel,std::list<FileOffset> >::const_iterator cell=typeCellOffsets[*type].begin();
cell!=typeCellOffsets[*type].end();
++cell) {
FileOffset bitmapCellOffset=bitmapOffset+
((cell->first.y-nodeTypeData[*type].cellYStart)*nodeTypeData[*type].cellXCount+
cell->first.x-nodeTypeData[*type].cellXStart)*dataOffsetBytes;
FileOffset previousOffset=0;
FileOffset cellOffset;
if (!writer.GetPos(cellOffset)) {
progress.Error("Cannot get cell start position in file");
return false;
}
if (!writer.SetPos(bitmapCellOffset)) {
progress.Error("Cannot go to cell start position in file");
return false;
}
writer.WriteFileOffset(cellOffset-dataStartOffset+1,
dataOffsetBytes);
if (!writer.SetPos(cellOffset)) {
progress.Error("Cannot go back to cell start position in file");
return false;
}
writer.WriteNumber((uint32_t)cell->second.size());
for (std::list<FileOffset>::const_iterator offset=cell->second.begin();
offset!=cell->second.end();
++offset) {
writer.WriteNumber((FileOffset)(*offset-previousOffset));
previousOffset=*offset;
}
}
}
}
return !writer.HasError() && writer.Close();
}
bool Import(const ImportParameter& parameter,
Progress& progress)
{
// TODO: verify parameter
TypeConfigRef typeConfig(new TypeConfig());
std::list<ImportModule*> modules;
if (parameter.GetAreaWayMinMag()<=parameter.GetOptimizationMaxMag()) {
progress.Error("Area way index minimum magnification is <= than optimization max magnification");
}
progress.SetStep("Loading type config");
if (!typeConfig->LoadFromOSTFile(parameter.GetTypefile())) {
progress.Error("Cannot load type configuration!");
return false;
}
progress.Info("Number of types: "+NumberToString(typeConfig->GetTypes().size()));
progress.Info("Number of node types: "+NumberToString(typeConfig->GetNodeTypes().size())+" "+NumberToString(typeConfig->GetNodeTypeIdBytes())+" byte(s)");
progress.Info("Number of way types: "+NumberToString(typeConfig->GetWayTypes().size())+" "+NumberToString(typeConfig->GetWayTypeIdBytes())+" byte(s)");
progress.Info("Number of area types: "+NumberToString(typeConfig->GetAreaTypes().size())+" "+NumberToString(typeConfig->GetAreaTypeIdBytes())+" byte(s)");
typeConfig->RegisterNameTag("name",0);
typeConfig->RegisterNameTag("place_name",1);
/*
typeConfig->RegisterNameAltTag("name:ru",0);
typeConfig->RegisterNameAltTag("place_name:ru",1);
*/
/* 1 */
modules.push_back(new TypeDataGenerator());
/* 2 */
modules.push_back(new Preprocess());
/* 3 */
modules.push_back(new RawNodeIndexGenerator(AppendFileToDir(parameter.GetDestinationDirectory(),
"rawnodes.dat"),
AppendFileToDir(parameter.GetDestinationDirectory(),
"rawnode.idx")));
/* 4 */
modules.push_back(new RawWayIndexGenerator(AppendFileToDir(parameter.GetDestinationDirectory(),
"rawways.dat"),
AppendFileToDir(parameter.GetDestinationDirectory(),
"rawway.idx")));
/* 5 */
modules.push_back(new RawRelationIndexGenerator(AppendFileToDir(parameter.GetDestinationDirectory(),
"rawrels.dat"),
AppendFileToDir(parameter.GetDestinationDirectory(),
"rawrel.idx")));
/* 6 */
modules.push_back(new RelAreaDataGenerator());
/* 7 */
modules.push_back(new WayAreaDataGenerator());
/* 8 */
modules.push_back(new MergeAreaDataGenerator());
/* 9 */
modules.push_back(new MergeAreasGenerator());
/* 10 */
modules.push_back(new WayWayDataGenerator());
/* 11 */
modules.push_back(new OptimizeAreaWayIdsGenerator());
/* 12 */
modules.push_back(new NodeDataGenerator());
/* 13 */
modules.push_back(new SortNodeDataGenerator());
/* 14 */
modules.push_back(new SortWayDataGenerator());
/* 15 */
modules.push_back(new AreaNodeIndexGenerator());
/* 16 */
modules.push_back(new AreaWayIndexGenerator());
/* 17 */
modules.push_back(new AreaAreaIndexGenerator());
/* 18 */
modules.push_back(new WaterIndexGenerator());
/* 19 */
modules.push_back(new OptimizeAreasLowZoomGenerator());
/* 20 */
modules.push_back(new OptimizeWaysLowZoomGenerator());
/* 21 */
modules.push_back(new LocationIndexGenerator());
/* 22 */
modules.push_back(new RouteDataGenerator());
/* 23 */
modules.push_back(new NumericIndexGenerator<Id,Intersection>(std::string("Generating '")+RoutingService::FILENAME_INTERSECTIONS_IDX+"'",
AppendFileToDir(parameter.GetDestinationDirectory(),
RoutingService::FILENAME_INTERSECTIONS_DAT),
AppendFileToDir(parameter.GetDestinationDirectory(),
RoutingService::FILENAME_INTERSECTIONS_IDX)));
#if defined(OSMSCOUT_IMPORT_HAVE_LIB_MARISA)
/* 24 */
modules.push_back(new TextIndexGenerator());
#endif
bool result=ExecuteModules(modules,
parameter,
progress,
typeConfig);
for (const auto& module : modules) {
delete module;
}
return result;
}
bool AreaWayIndexGenerator::CalculateDistribution(const TypeConfigRef& typeConfig,
const ImportParameter& parameter,
Progress& progress,
std::vector<TypeData>& wayTypeData,
size_t& maxLevel) const
{
FileScanner wayScanner;
TypeInfoSet remainingWayTypes;
size_t level;
maxLevel=0;
wayTypeData.resize(typeConfig->GetTypeCount());
if (!wayScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"ways.dat"),
FileScanner::Sequential,
parameter.GetWayDataMemoryMaped())) {
progress.Error("Cannot open 'ways.dat'");
return false;
}
remainingWayTypes.Set(typeConfig->GetWayTypes());
level=parameter.GetAreaWayMinMag();
while (!remainingWayTypes.Empty()) {
uint32_t wayCount=0;
TypeInfoSet currentWayTypes(remainingWayTypes);
double cellWidth=360.0/pow(2.0,(int)level);
double cellHeight=180.0/pow(2.0,(int)level);
std::vector<CoordCountMap> cellFillCount(typeConfig->GetTypeCount());
progress.Info("Scanning Level "+NumberToString(level)+" ("+NumberToString(remainingWayTypes.Size())+" types remaining)");
wayScanner.GotoBegin();
if (!wayScanner.Read(wayCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
Way way;
for (uint32_t w=1; w<=wayCount; w++) {
progress.SetProgress(w,wayCount);
if (!way.Read(*typeConfig,
wayScanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(w)+" of "+
NumberToString(wayCount)+
" in file '"+
wayScanner.GetFilename()+"'");
return false;
}
// Count number of entries per current type and coordinate
if (!currentWayTypes.IsSet(way.GetType())) {
continue;
}
GeoBox boundingBox;
way.GetBoundingBox(boundingBox);
//
// Calculate minimum and maximum tile ids that are covered
// by the way
// Renormalized coordinate space (everything is >=0)
//
uint32_t minxc=(uint32_t)floor((boundingBox.GetMinLon()+180.0)/cellWidth);
uint32_t maxxc=(uint32_t)floor((boundingBox.GetMaxLon()+180.0)/cellWidth);
uint32_t minyc=(uint32_t)floor((boundingBox.GetMinLat()+90.0)/cellHeight);
uint32_t maxyc=(uint32_t)floor((boundingBox.GetMaxLat()+90.0)/cellHeight);
for (uint32_t y=minyc; y<=maxyc; y++) {
for (uint32_t x=minxc; x<=maxxc; x++) {
cellFillCount[way.GetType()->GetIndex()][Pixel(x,y)]++;
}
}
}
// Check if cell fill for current type is in defined limits
for (auto &type : currentWayTypes) {
size_t i=type->GetIndex();
CalculateStatistics(level,
wayTypeData[i],
cellFillCount[i]);
if (!FitsIndexCriteria(parameter,
progress,
*typeConfig->GetTypeInfo(i),
wayTypeData[i],
cellFillCount[i])) {
currentWayTypes.Remove(type);
}
}
for (const auto &type : currentWayTypes) {
maxLevel=std::max(maxLevel,level);
progress.Info("Type "+type->GetName()+", "+NumberToString(wayTypeData[type->GetIndex()].indexCells)+" cells, "+NumberToString(wayTypeData[type->GetIndex()].indexEntries)+" objects");
remainingWayTypes.Remove(type);
}
level++;
}
return !wayScanner.HasError() && wayScanner.Close();
}