bool LocationIndex::VisitLocationAddresses(const AdminRegion& region, const Location& location, AddressVisitor& visitor) const { FileScanner scanner; bool stopped=false; if (!scanner.Open(AppendFileToDir(path, FILENAME_LOCATION_IDX), FileScanner::LowMemRandom, true)) { log.Error() << "Cannot open file '" << scanner.GetFilename() << "'!"; return false; } if (!scanner.SetPos(indexOffset)) { return false; } if (!VisitLocationAddressEntries(scanner, region, location, visitor, stopped)) { return false; } return !scanner.HasError() && scanner.Close(); }
bool LocationIndex::ResolveAdminRegionHierachie(const AdminRegionRef& adminRegion, std::map<FileOffset,AdminRegionRef >& refs) const { FileScanner scanner; if (!scanner.Open(AppendFileToDir(path, FILENAME_LOCATION_IDX), FileScanner::LowMemRandom, true)) { log.Error() << "Cannot open file '" << scanner.GetFilename() << "'!"; return false; } if (!scanner.SetPos(indexOffset)) { return false; } std::list<FileOffset> offsets; refs[adminRegion->regionOffset]=adminRegion; if (adminRegion->parentRegionOffset!=0) { offsets.push_back(adminRegion->parentRegionOffset); } while (!offsets.empty()) { std::list<FileOffset> newOffsets; for (const auto& offset : offsets) { if (refs.find(offset)!=refs.end()) { continue; } if (!scanner.SetPos(offset)) { return false; } AdminRegion adminRegion; if (!LoadAdminRegion(scanner, adminRegion)) { return false; } refs[adminRegion.regionOffset]=std::make_shared<AdminRegion>(adminRegion); if (adminRegion.parentRegionOffset!=0) { newOffsets.push_back(adminRegion.parentRegionOffset); } } offsets.clear(); std::swap(offsets, newOffsets); } return !scanner.HasError() && scanner.Close(); }
bool LocationIndex::Load(const std::string& path) { this->path=path; FileScanner scanner; if (!scanner.Open(AppendFileToDir(path, FILENAME_LOCATION_IDX), FileScanner::LowMemRandom, true)) { log.Error() << "Cannot open file '" << scanner.GetFilename() << "'!"; return false; } if (!(scanner.Read(bytesForNodeFileOffset) && scanner.Read(bytesForAreaFileOffset) && scanner.Read(bytesForWayFileOffset))) { return false; } uint32_t ignoreTokenCount; if (!scanner.ReadNumber(ignoreTokenCount)) { return false; } for (size_t i=0; i<ignoreTokenCount; i++) { std::string token; if (!scanner.Read(token)) { return false; } regionIgnoreTokens.insert(token); } if (!scanner.ReadNumber(ignoreTokenCount)) { return false; } for (size_t i=0; i<ignoreTokenCount; i++) { std::string token; if (!scanner.Read(token)) { return false; } locationIgnoreTokens.insert(token); } if (!scanner.GetPos(indexOffset)) { return false; } return !scanner.HasError() && scanner.Close(); }
bool DebugDatabase::ResolveReferences(const std::string& mapName, RefType fileType, const std::set<ObjectOSMRef>& ids, const std::set<ObjectFileRef>& fileOffsets, std::map<ObjectOSMRef,ObjectFileRef>& idFileOffsetMap, std::map<ObjectFileRef,ObjectOSMRef>& fileOffsetIdMap) { FileScanner scanner; uint32_t entryCount; std::string filename=AppendFileToDir(path,mapName); if (!scanner.Open(filename,FileScanner::LowMemRandom,false)) { std::cerr << "Cannot open file '" << scanner.GetFilename() << "'!" << std::endl; return false; } if (!scanner.Read(entryCount)) { return false; } for (size_t i=1; i<=entryCount; i++) { Id id; uint8_t typeByte; OSMRefType osmType; FileOffset fileOffset; if (!scanner.Read(id)) { return false; } if (!scanner.Read(typeByte)) { return false; } osmType=(OSMRefType)typeByte; if (!scanner.ReadFileOffset(fileOffset)) { return false; } ObjectOSMRef osmRef(id,osmType); ObjectFileRef fileRef(fileOffset,fileType); if (ids.find(osmRef)!=ids.end() || fileOffsets.find(fileRef)!=fileOffsets.end()) { idFileOffsetMap.insert(std::make_pair(osmRef,fileRef)); fileOffsetIdMap.insert(std::make_pair(fileRef,osmRef)); } } return scanner.Close(); }
bool LocationIndex::VisitAdminRegions(AdminRegionVisitor& visitor) const { FileScanner scanner; if (!scanner.Open(AppendFileToDir(path, FILENAME_LOCATION_IDX), FileScanner::LowMemRandom, false)) { log.Error() << "Cannot open file '" << scanner.GetFilename() << "'!"; return false; } if (!scanner.SetPos(indexOffset)) { return false; } uint32_t regionCount; if (!scanner.ReadNumber(regionCount)) { return false; } for (size_t i=0; i<regionCount; i++) { AdminRegionVisitor::Action action; FileOffset nextChildOffset; if (!scanner.ReadFileOffset(nextChildOffset)) { return false; } action=VisitRegionEntries(scanner, visitor); if (action==AdminRegionVisitor::error) { return false; } else if (action==AdminRegionVisitor::stop) { return true; } else if (action==AdminRegionVisitor::skipChildren) { if (i+1<regionCount) { if (!scanner.SetPos(nextChildOffset)) { return false; } } } } return !scanner.HasError() && scanner.Close(); }
/** * Load the object variant data from the given file. * * @param typeConfig * TypeConfig instance * @param filename * Name of the file containing the object variant data * @return * True on success, else false */ bool ObjectVariantDataFile::Load(const TypeConfig& typeConfig, const std::string& filename) { FileScanner scanner; data.clear(); isLoaded=false; this->filename=filename; try { scanner.Open(filename, FileScanner::Sequential,true); uint32_t dataCount; scanner.Read(dataCount); data.resize(dataCount); for (size_t i=0; i<dataCount; i++) { data[i].Read(typeConfig, scanner); } scanner.Close(); isLoaded=true; } catch (IOException& e) { log.Error() << e.GetDescription(); scanner.CloseFailsafe(); return false; } return true; }
bool Database::Open(const std::string& path) { assert(!path.empty()); this->path=path; typeConfig=new TypeConfig(); if (!typeConfig->LoadFromDataFile(path)) { log.Error() << "Cannot load 'types.dat'!"; return false; } FileScanner scanner; std::string file=AppendFileToDir(path,"bounding.dat"); if (!scanner.Open(file,FileScanner::Normal,true)) { log.Error() << "Cannot open '" << scanner.GetFilename() << "'"; return false; } if (!scanner.ReadBox(boundingBox)) { log.Error() << "Error while reading '" << scanner.GetFilename() << "'"; } log.Debug() << "BoundingBox: " << boundingBox.GetDisplayText(); if (scanner.HasError() || !scanner.Close()) { log.Error() << "Cannot while reading/closing '" << scanner.GetFilename() << "'"; return false; } isOpen=true; return true; }
bool Database::Open(const std::string& path) { assert(!path.empty()); this->path=path; typeConfig=std::make_shared<TypeConfig>(); if (!typeConfig->LoadFromDataFile(path)) { log.Error() << "Cannot load 'types.dat'!"; return false; } FileScanner scanner; try { scanner.Open(AppendFileToDir(path,"bounding.dat"), FileScanner::Normal, false); scanner.ReadBox(boundingBox); log.Debug() << "BoundingBox: " << boundingBox.GetDisplayText(); scanner.Close(); } catch (IOException& e) { log.Error() << e.GetDescription(); scanner.CloseFailsafe(); return false; } isOpen=true; return true; }
bool OptimizeAreaWayIdsGenerator::ScanAreaIds(const ImportParameter& parameter, Progress& progress, const TypeConfig& typeConfig, NodeUseMap& nodeUseMap) { FileScanner scanner; uint32_t dataCount=0; progress.SetAction("Scanning ids from 'areas2.tmp'"); if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "areas2.tmp"), FileScanner::Sequential, parameter.GetAreaDataMemoryMaped())) { progress.Error(std::string("Cannot open '")+scanner.GetFilename()+"'"); return false; } if (!scanner.Read(dataCount)) { progress.Error("Error while reading number of data entries in file"); return false; } for (uint32_t current=1; current<=dataCount; current++) { uint8_t type; Id id; Area data; progress.SetProgress(current,dataCount); if (!scanner.Read(type) || !scanner.Read(id) || !data.ReadImport(typeConfig, scanner)) { progress.Error(std::string("Error while reading data entry ")+ NumberToString(current)+" of "+ NumberToString(dataCount)+ " in file '"+ scanner.GetFilename()+"'"); return false; } for (const auto& ring: data.rings) { std::unordered_set<Id> nodeIds; if (!ring.GetType()->CanRoute()) { continue; } for (const auto id: ring.ids) { if (nodeIds.find(id)==nodeIds.end()) { nodeUseMap.SetNodeUsed(id); nodeIds.insert(id); } } } } if (!scanner.Close()) { progress.Error(std::string("Error while closing file '")+ scanner.GetFilename()+"'"); return false; } return true; }
bool OptimizeAreaWayIdsGenerator::CopyWays(const ImportParameter& parameter, Progress& progress, const TypeConfig& typeConfig, NodeUseMap& nodeUseMap) { FileScanner scanner; FileWriter writer; uint32_t dataCount=0; progress.SetAction("Copy data from 'wayway.tmp' to 'ways.tmp'"); if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "wayway.tmp"), FileScanner::Sequential, parameter.GetWayDataMemoryMaped())) { progress.Error(std::string("Cannot open '")+scanner.GetFilename()+"'"); return false; } if (!scanner.Read(dataCount)) { progress.Error("Error while reading number of data entries in file"); return false; } if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "ways.tmp"))) { progress.Error(std::string("Cannot create '")+writer.GetFilename()+"'"); return false; } writer.Write(dataCount); for (uint32_t current=1; current<=dataCount; current++) { uint8_t type; Id id; Way data; progress.SetProgress(current,dataCount); if (!scanner.Read(type) || !scanner.Read(id) || !data.Read(typeConfig, scanner)) { progress.Error(std::string("Error while reading data entry ")+ NumberToString(current)+" of "+ NumberToString(dataCount)+ " in file '"+ scanner.GetFilename()+"'"); return false; } for (auto& id : data.ids) { if (!nodeUseMap.IsNodeUsedAtLeastTwice(id)) { id=0; } } if (!writer.Write(type) || !writer.Write(id) || !data.Write(typeConfig, writer)) { progress.Error(std::string("Error while writing data entry to file '")+ writer.GetFilename()+"'"); return false; } } if (!scanner.Close()) { progress.Error(std::string("Error while closing file '")+ scanner.GetFilename()+"'"); return false; } if (!writer.Close()) { progress.Error(std::string("Error while closing file '")+ writer.GetFilename()+"'"); return false; } return true; }
bool OptimizeAreaWayIdsGenerator::ScanWayIds(const ImportParameter& parameter, Progress& progress, const TypeConfig& typeConfig, NodeUseMap& nodeUseMap) { FileScanner scanner; uint32_t dataCount=0; progress.SetAction("Scanning ids from 'wayway.tmp'"); if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "wayway.tmp"), FileScanner::Sequential, parameter.GetWayDataMemoryMaped())) { progress.Error(std::string("Cannot open '")+scanner.GetFilename()+"'"); return false; } if (!scanner.Read(dataCount)) { progress.Error("Error while reading number of data entries in file"); return false; } for (uint32_t current=1; current<=dataCount; current++) { uint8_t type; Id id; Way data; progress.SetProgress(current,dataCount); if (!scanner.Read(type) || !scanner.Read(id) || !data.Read(typeConfig, scanner)) { progress.Error(std::string("Error while reading data entry ")+ NumberToString(current)+" of "+ NumberToString(dataCount)+ " in file '"+ scanner.GetFilename()+"'"); return false; } if (!data.GetType()->CanRoute()) { continue; } std::unordered_set<Id> nodeIds; for (const auto& id : data.ids) { if (nodeIds.find(id)==nodeIds.end()) { nodeUseMap.SetNodeUsed(id); nodeIds.insert(id); } } // If we have a circular way, we "fake" a double usage, // to make sure, that the node id of the first node // is not dropped later on, and we cannot detect // circular ways anymore if (data.ids.front()==data.ids.back()) { nodeUseMap.SetNodeUsed(data.ids.back()); } } if (!scanner.Close()) { progress.Error(std::string("Error while closing file '")+ scanner.GetFilename()+"'"); return false; } return true; }
bool AreaNodeIndexGenerator::Import(const ImportParameter& parameter, Progress& progress, const TypeConfig& typeConfig) { FileScanner nodeScanner; FileWriter writer; std::set<TypeId> remainingNodeTypes; //! Set of types we still must process std::vector<TypeData> nodeTypeData; size_t level; size_t maxLevel=0; nodeTypeData.resize(typeConfig.GetTypes().size()); if (!nodeScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "nodes.dat"), FileScanner::Sequential, true)) { progress.Error("Cannot open 'nodes.dat'"); return false; } // // Scanning distribution // progress.SetAction("Scanning level distribution of node types"); // Initially we must process all types that represents nodes and that should // not be ignored for (size_t i=0; i<typeConfig.GetTypes().size(); i++) { if (typeConfig.GetTypeInfo(i).CanBeNode() && !typeConfig.GetTypeInfo(i).GetIgnore()) { remainingNodeTypes.insert(i); } } level=parameter.GetAreaNodeMinMag(); while (!remainingNodeTypes.empty()) { uint32_t nodeCount=0; std::set<TypeId> currentNodeTypes(remainingNodeTypes); double cellWidth=360.0/pow(2.0,(int)level); double cellHeight=180.0/pow(2.0,(int)level); std::vector<std::map<Pixel,size_t> > cellFillCount; cellFillCount.resize(typeConfig.GetTypes().size()); progress.Info("Scanning Level "+NumberToString(level)+" ("+NumberToString(remainingNodeTypes.size())+" types still to process)"); nodeScanner.GotoBegin(); if (!nodeScanner.Read(nodeCount)) { progress.Error("Error while reading number of data entries in file"); return false; } for (uint32_t n=1; n<=nodeCount; n++) { progress.SetProgress(n,nodeCount); FileOffset offset; Node node; nodeScanner.GetPos(offset); if (!node.Read(nodeScanner)) { progress.Error(std::string("Error while reading data entry ")+ NumberToString(n)+" of "+ NumberToString(nodeCount)+ " in file '"+ nodeScanner.GetFilename()+"'"); return false; } // If we still need to handle this type, // count number of entries per type and tile cell if (currentNodeTypes.find(node.GetType())!=currentNodeTypes.end()) { uint32_t xc=(uint32_t)floor((node.GetLon()+180.0)/cellWidth); uint32_t yc=(uint32_t)floor((node.GetLat()+90.0)/cellHeight); cellFillCount[node.GetType()][Pixel(xc,yc)]++; } } // Check statistics for each type // If statistics are within goal limits, use this level // for this type (else try again with the next higher level) for (size_t i=0; i<typeConfig.GetTypes().size(); i++) { if (currentNodeTypes.find(i)!=currentNodeTypes.end()) { size_t entryCount=0; size_t max=0; nodeTypeData[i].indexLevel=(uint32_t)level; nodeTypeData[i].indexCells=cellFillCount[i].size(); nodeTypeData[i].indexEntries=0; if (!cellFillCount[i].empty()) { nodeTypeData[i].cellXStart=cellFillCount[i].begin()->first.x; nodeTypeData[i].cellYStart=cellFillCount[i].begin()->first.y; nodeTypeData[i].cellXEnd=nodeTypeData[i].cellXStart; nodeTypeData[i].cellYEnd=nodeTypeData[i].cellYStart; for (std::map<Pixel,size_t>::const_iterator cell=cellFillCount[i].begin(); cell!=cellFillCount[i].end(); ++cell) { nodeTypeData[i].indexEntries+=cell->second; nodeTypeData[i].cellXStart=std::min(nodeTypeData[i].cellXStart,cell->first.x); nodeTypeData[i].cellXEnd=std::max(nodeTypeData[i].cellXEnd,cell->first.x); nodeTypeData[i].cellYStart=std::min(nodeTypeData[i].cellYStart,cell->first.y); nodeTypeData[i].cellYEnd=std::max(nodeTypeData[i].cellYEnd,cell->first.y); } } nodeTypeData[i].cellXCount=nodeTypeData[i].cellXEnd-nodeTypeData[i].cellXStart+1; nodeTypeData[i].cellYCount=nodeTypeData[i].cellYEnd-nodeTypeData[i].cellYStart+1; // Count absolute number of entries for (std::map<Pixel,size_t>::const_iterator cell=cellFillCount[i].begin(); cell!=cellFillCount[i].end(); ++cell) { entryCount+=cell->second; max=std::max(max,cell->second); } // Average number of entries per tile cell double average=entryCount*1.0/cellFillCount[i].size(); // If we do not have any entries, we store it now if (cellFillCount[i].empty()) { continue; } // If the fill rate of the index is too low, we use this index level anyway if (nodeTypeData[i].indexCells/(1.0*nodeTypeData[i].cellXCount*nodeTypeData[i].cellYCount)<= parameter.GetAreaNodeIndexMinFillRate()) { progress.Warning(typeConfig.GetTypeInfo(i).GetName()+" ("+NumberToString(i)+") is not well distributed"); continue; } // If average fill size and max fill size for tile cells // is within limits, store it now. if (max<=parameter.GetAreaNodeIndexCellSizeMax() && average<=parameter.GetAreaNodeIndexCellSizeAverage()) { continue; } // else, we remove it from the list and try again with an higher // level. currentNodeTypes.erase(i); } } // Now process all types for this limit, that are within the limits for (std::set<TypeId>::const_iterator cnt=currentNodeTypes.begin(); cnt!=currentNodeTypes.end(); cnt++) { maxLevel=std::max(maxLevel,level); progress.Info("Type "+typeConfig.GetTypeInfo(*cnt).GetName()+"(" + NumberToString(*cnt)+"), "+NumberToString(nodeTypeData[*cnt].indexCells)+" cells, "+NumberToString(nodeTypeData[*cnt].indexEntries)+" objects"); remainingNodeTypes.erase(*cnt); } level++; } // // Writing index file // progress.SetAction("Generating 'areanode.idx'"); if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "areanode.idx"))) { progress.Error("Cannot create 'areanode.idx'"); return false; } uint32_t indexEntries=0; // Count number of types in index for (size_t i=0; i<typeConfig.GetTypes().size(); i++) { if (typeConfig.GetTypeInfo(i).CanBeNode() && nodeTypeData[i].HasEntries()) { indexEntries++; } } writer.Write(indexEntries); // Store index data for each type for (size_t i=0; i<typeConfig.GetTypes().size(); i++) { if (typeConfig.GetTypeInfo(i).CanBeNode() && nodeTypeData[i].HasEntries()) { FileOffset bitmapOffset=0; uint8_t dataOffsetBytes=0; writer.WriteNumber(typeConfig.GetTypeInfo(i).GetId()); writer.GetPos(nodeTypeData[i].indexOffset); writer.WriteFileOffset(bitmapOffset); writer.Write(dataOffsetBytes); writer.WriteNumber(nodeTypeData[i].indexLevel); writer.WriteNumber(nodeTypeData[i].cellXStart); writer.WriteNumber(nodeTypeData[i].cellXEnd); writer.WriteNumber(nodeTypeData[i].cellYStart); writer.WriteNumber(nodeTypeData[i].cellYEnd); } } // Now store index bitmap for each type in increasing level order (why?) for (size_t l=0; l<=maxLevel; l++) { std::set<TypeId> indexTypes; uint32_t nodeCount; double cellWidth=360.0/pow(2.0,(int)l); double cellHeight=180.0/pow(2.0,(int)l); for (size_t i=0; i<typeConfig.GetTypes().size(); i++) { if (typeConfig.GetTypeInfo(i).CanBeNode() && nodeTypeData[i].HasEntries() && nodeTypeData[i].indexLevel==l) { indexTypes.insert(i); } } if (indexTypes.empty()) { continue; } progress.Info("Scanning nodes for index level "+NumberToString(l)); std::vector<std::map<Pixel,std::list<FileOffset> > > typeCellOffsets; typeCellOffsets.resize(typeConfig.GetTypes().size()); nodeScanner.GotoBegin(); if (!nodeScanner.Read(nodeCount)) { progress.Error("Error while reading number of data entries in file"); return false; } // // Collect all offsets // for (uint32_t n=1; n<=nodeCount; n++) { progress.SetProgress(n,nodeCount); FileOffset offset; Node node; nodeScanner.GetPos(offset); if (!node.Read(nodeScanner)) { progress.Error(std::string("Error while reading data entry ")+ NumberToString(n)+" of "+ NumberToString(nodeCount)+ " in file '"+ nodeScanner.GetFilename()+"'"); return false; } if (indexTypes.find(node.GetType())!=indexTypes.end()) { uint32_t xc=(uint32_t)floor((node.GetLon()+180.0)/cellWidth); uint32_t yc=(uint32_t)floor((node.GetLat()+90.0)/cellHeight); typeCellOffsets[node.GetType()][Pixel(xc,yc)].push_back(offset); } } // // Write bitmap // for (std::set<TypeId>::const_iterator type=indexTypes.begin(); type!=indexTypes.end(); ++type) { size_t indexEntries=0; size_t dataSize=0; char buffer[10]; for (std::map<Pixel,std::list<FileOffset> >::const_iterator cell=typeCellOffsets[*type].begin(); cell!=typeCellOffsets[*type].end(); ++cell) { indexEntries+=cell->second.size(); dataSize+=EncodeNumber(cell->second.size(),buffer); FileOffset previousOffset=0; for (std::list<FileOffset>::const_iterator offset=cell->second.begin(); offset!=cell->second.end(); ++offset) { FileOffset data=*offset-previousOffset; dataSize+=EncodeNumber(data,buffer); previousOffset=*offset; } } // "+1" because we add +1 to every offset, to generate offset > 0 uint8_t dataOffsetBytes=BytesNeededToAddressFileData(dataSize); progress.Info("Writing map for "+ typeConfig.GetTypeInfo(*type).GetName()+", "+ NumberToString(typeCellOffsets[*type].size())+" cells, "+ NumberToString(indexEntries)+" entries, "+ ByteSizeToString(1.0*dataOffsetBytes*nodeTypeData[*type].cellXCount*nodeTypeData[*type].cellYCount)); FileOffset bitmapOffset; if (!writer.GetPos(bitmapOffset)) { progress.Error("Cannot get type index start position in file"); return false; } assert(nodeTypeData[*type].indexOffset!=0); if (!writer.SetPos(nodeTypeData[*type].indexOffset)) { progress.Error("Cannot go to type index offset in file"); return false; } writer.WriteFileOffset(bitmapOffset); writer.Write(dataOffsetBytes); if (!writer.SetPos(bitmapOffset)) { progress.Error("Cannot go to type index start position in file"); return false; } // Write the bitmap with offsets for each cell // We prefill with zero and only overwrite cells that have data // So zero means "no data for this cell" for (size_t i=0; i<nodeTypeData[*type].cellXCount*nodeTypeData[*type].cellYCount; i++) { FileOffset cellOffset=0; writer.WriteFileOffset(cellOffset, dataOffsetBytes); } FileOffset dataStartOffset; if (!writer.GetPos(dataStartOffset)) { progress.Error("Cannot get start of data section after bitmap"); return false; } // Now write the list of offsets of objects for every cell with content for (std::map<Pixel,std::list<FileOffset> >::const_iterator cell=typeCellOffsets[*type].begin(); cell!=typeCellOffsets[*type].end(); ++cell) { FileOffset bitmapCellOffset=bitmapOffset+ ((cell->first.y-nodeTypeData[*type].cellYStart)*nodeTypeData[*type].cellXCount+ cell->first.x-nodeTypeData[*type].cellXStart)*dataOffsetBytes; FileOffset previousOffset=0; FileOffset cellOffset; if (!writer.GetPos(cellOffset)) { progress.Error("Cannot get cell start position in file"); return false; } if (!writer.SetPos(bitmapCellOffset)) { progress.Error("Cannot go to cell start position in file"); return false; } writer.WriteFileOffset(cellOffset-dataStartOffset+1, dataOffsetBytes); if (!writer.SetPos(cellOffset)) { progress.Error("Cannot go back to cell start position in file"); return false; } writer.WriteNumber((uint32_t)cell->second.size()); for (std::list<FileOffset>::const_iterator offset=cell->second.begin(); offset!=cell->second.end(); ++offset) { writer.WriteNumber((FileOffset)(*offset-previousOffset)); previousOffset=*offset; } } } } return !writer.HasError() && writer.Close(); }
bool OptimizeAreasLowZoomGenerator::HandleAreas(const ImportParameter& parameter, Progress& progress, const TypeConfig& typeConfig, FileWriter& writer, const TypeInfoSet& types, std::list<TypeData>& typesData) { FileScanner scanner; // Everything smaller than 2mm should get dropped. Width, height and DPI come from the Nexus 4 double dpi=320.0; double pixel=2.0/* mm */ * dpi / 25.4 /* inch */; progress.Info("Minimum visible size in pixel: "+NumberToString((unsigned long)pixel)); try { scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(), AreaDataFile::AREAS_DAT), FileScanner::Sequential, parameter.GetWayDataMemoryMaped()); TypeInfoSet typesToProcess(types); std::vector<std::list<AreaRef> > allAreas(typeConfig.GetTypeCount()); while (true) { // // Load type data // TypeInfoSet loadedTypes; if (!GetAreas(typeConfig, parameter, progress, scanner, typesToProcess, allAreas, loadedTypes)) { return false; } typesToProcess.Remove(loadedTypes); for (const auto& type : loadedTypes) { progress.SetAction("Optimizing type "+ type->GetName()); for (uint32_t level=parameter.GetOptimizationMinMag(); level<=parameter.GetOptimizationMaxMag(); level++) { Magnification magnification; // Magnification, we optimize for std::list<AreaRef> optimizedAreas; magnification.SetLevel(level); OptimizeAreas(allAreas[type->GetIndex()], optimizedAreas, 1280,768, dpi, pixel, magnification, parameter.GetOptimizationWayMethod()); if (optimizedAreas.empty()) { progress.Debug("Empty optimization result for level "+NumberToString(level)+", no index generated"); TypeData typeData; typeData.type=type; typeData.optLevel=level; typesData.push_back(typeData); continue; } progress.Info("Optimized from "+NumberToString(allAreas[type->GetIndex()].size())+" to "+NumberToString(optimizedAreas.size())+" areas"); /* size_t optAreas=optimizedAreas.size(); size_t optRoles=0; size_t optNodes=0; for (std::list<AreaRef>::const_iterator a=optimizedAreas.begin(); a!=optimizedAreas.end(); ++a) { AreaRef area=*a; optRoles+=area->rings.size(); for (size_t r=0; r<area->rings.size(); r++) { optNodes+=area->rings[r].nodes.size(); } }*/ /* std::cout << "Areas: " << origAreas << " => " << optAreas << std::endl; std::cout << "Roles: " << origRoles << " => " << optRoles << std::endl; std::cout << "Nodes: " << origNodes << " => " << optNodes << std::endl;*/ TypeData typeData; typeData.type=type; typeData.optLevel=level; GetAreaIndexLevel(parameter, optimizedAreas, typeData); //std::cout << "Resulting index level: " << typeData.indexLevel << ", " << typeData.indexCells << ", " << typeData.indexEntries << std::endl; FileOffsetFileOffsetMap offsets; WriteAreas(typeConfig, writer, optimizedAreas, offsets); if (!WriteAreaBitmap(progress, writer, optimizedAreas, offsets, typeData)) { return false; } typesData.push_back(typeData); } allAreas[type->GetIndex()].clear(); } if (typesToProcess.Empty()) { break; } } scanner.Close(); } catch (IOException& e) { progress.Error(e.GetDescription()); return false; } return true; }
bool AreaWayIndexGenerator::Import(const TypeConfigRef& typeConfig, const ImportParameter& parameter, Progress& progress) { FileScanner wayScanner; FileWriter writer; std::vector<TypeData> wayTypeData; size_t maxLevel; progress.Info("Minimum magnification: "+NumberToString(parameter.GetAreaWayMinMag())); // // Scanning distribution // progress.SetAction("Scanning level distribution of way types"); if (!CalculateDistribution(typeConfig, parameter, progress, wayTypeData, maxLevel)) { return false; } // Calculate number of types which have data uint32_t indexEntries=0; for (const auto& type : typeConfig->GetWayTypes()) { if (wayTypeData[type->GetIndex()].HasEntries()) { indexEntries++; } } // // Writing index file // progress.SetAction("Generating 'areaway.idx'"); if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "areaway.idx"))) { progress.Error("Cannot create 'areaway.idx'"); return false; } writer.Write(indexEntries); for (const auto &type : typeConfig->GetWayTypes()) { size_t i=type->GetIndex(); if (wayTypeData[i].HasEntries()) { uint8_t dataOffsetBytes=0; FileOffset bitmapOffset=0; writer.WriteTypeId(type->GetWayId(), typeConfig->GetWayTypeIdBytes()); writer.GetPos(wayTypeData[i].indexOffset); writer.WriteFileOffset(bitmapOffset); writer.Write(dataOffsetBytes); writer.WriteNumber(wayTypeData[i].indexLevel); writer.WriteNumber(wayTypeData[i].cellXStart); writer.WriteNumber(wayTypeData[i].cellXEnd); writer.WriteNumber(wayTypeData[i].cellYStart); writer.WriteNumber(wayTypeData[i].cellYEnd); } } if (!wayScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "ways.dat"), FileScanner::Sequential, parameter.GetWayDataMemoryMaped())) { progress.Error("Cannot open 'ways.dat'"); return false; } for (size_t l=parameter.GetAreaWayMinMag(); l<=maxLevel; l++) { TypeInfoSet indexTypes(*typeConfig); uint32_t wayCount; double cellWidth=360.0/pow(2.0,(int)l); double cellHeight=180.0/pow(2.0,(int)l); wayScanner.GotoBegin(); for (const auto &type : typeConfig->GetWayTypes()) { if (wayTypeData[type->GetIndex()].HasEntries() && wayTypeData[type->GetIndex()].indexLevel==l) { indexTypes.Set(type); } } if (indexTypes.Empty()) { continue; } progress.Info("Scanning ways for index level "+NumberToString(l)); std::vector<CoordOffsetsMap> typeCellOffsets(typeConfig->GetTypeCount()); if (!wayScanner.Read(wayCount)) { progress.Error("Error while reading number of data entries in file"); return false; } Way way; for (uint32_t w=1; w<=wayCount; w++) { progress.SetProgress(w,wayCount); FileOffset offset; wayScanner.GetPos(offset); if (!way.Read(*typeConfig, wayScanner)) { progress.Error(std::string("Error while reading data entry ")+ NumberToString(w)+" of "+ NumberToString(wayCount)+ " in file '"+ wayScanner.GetFilename()+"'"); return false; } if (!indexTypes.IsSet(way.GetType())) { continue; } GeoBox boundingBox; way.GetBoundingBox(boundingBox); // // Calculate minimum and maximum tile ids that are covered // by the way // Renormalized coordinate space (everything is >=0) // uint32_t minxc=(uint32_t)floor((boundingBox.GetMinLon()+180.0)/cellWidth); uint32_t maxxc=(uint32_t)floor((boundingBox.GetMaxLon()+180.0)/cellWidth); uint32_t minyc=(uint32_t)floor((boundingBox.GetMinLat()+90.0)/cellHeight); uint32_t maxyc=(uint32_t)floor((boundingBox.GetMaxLat()+90.0)/cellHeight); for (uint32_t y=minyc; y<=maxyc; y++) { for (uint32_t x=minxc; x<=maxxc; x++) { typeCellOffsets[way.GetType()->GetIndex()][Pixel(x,y)].push_back(offset); } } } for (const auto &type : indexTypes) { size_t index=type->GetIndex(); if (!WriteBitmap(progress, writer, *typeConfig->GetTypeInfo(index), wayTypeData[index], typeCellOffsets[index])) { return false; } } } return !writer.HasError() && writer.Close(); }
bool MergeAreasGenerator::Import(const TypeConfigRef& typeConfig, const ImportParameter& parameter, Progress& progress) { TypeInfoSet mergeTypes; FileScanner scanner; FileWriter writer; uint32_t areasWritten=0; for (const auto& type : typeConfig->GetTypes()) { if (type->CanBeArea() && type->GetMergeAreas()) { mergeTypes.Set(type); } } std::unordered_set<Id> nodeUseMap; try { scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(), MergeAreaDataGenerator::AREAS_TMP), FileScanner::Sequential, parameter.GetRawWayDataMemoryMaped()); if (!ScanAreaNodeIds(progress, *typeConfig, scanner, mergeTypes, nodeUseMap)) { return false; } uint32_t nodeCount=nodeUseMap.size(); progress.Info("Found "+NumberToString(nodeCount)+" nodes as possible connection points for areas"); /* ------ */ writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(), AREAS2_TMP)); writer.Write(areasWritten); while (true) { TypeInfoSet loadedTypes; std::vector<AreaMergeData> mergeJob(typeConfig->GetTypeCount()); // // Load type data // progress.SetAction("Collecting area data by type"); if (!GetAreas(parameter, progress, *typeConfig, mergeTypes, loadedTypes, nodeUseMap, scanner, writer, mergeJob, areasWritten)) { return false; } // Merge progress.SetAction("Merging areas"); for (const auto& type : loadedTypes) { if (!mergeJob[type->GetIndex()].areas.empty()) { progress.Info("Merging areas of type "+type->GetName()); MergeAreas(progress, nodeUseMap, mergeJob[type->GetIndex()]); progress.Info("Reduced areas of '"+type->GetName()+"' from "+NumberToString(mergeJob[type->GetIndex()].areaCount)+" to "+NumberToString(mergeJob[type->GetIndex()].areaCount-mergeJob[type->GetIndex()].mergedAway.size())); mergeJob[type->GetIndex()].areas.clear(); } } // Store back merge result if (!loadedTypes.Empty()) { if (!WriteMergeResult(progress, *typeConfig, scanner, writer, loadedTypes, mergeJob, areasWritten)) { return false; } mergeTypes.Remove(loadedTypes); } if (mergeTypes.Empty()) { break; } } scanner.Close(); writer.GotoBegin(); writer.Write(areasWritten); writer.Close(); } catch (IOException& e) { progress.Error(e.GetDescription()); scanner.CloseFailsafe(); writer.CloseFailsafe(); return false; } return true; }
bool AreaAreaIndexGenerator::Import(const TypeConfigRef& typeConfig, const ImportParameter& parameter, Progress& progress) { FileScanner scanner; size_t areas=0; // Number of areas found size_t areasConsumed=0; // Number of areas consumed std::vector<double> cellWidth; std::vector<double> cellHeight; std::map<Pixel,AreaLeaf> leafs; std::map<Pixel,AreaLeaf> newAreaLeafs; cellWidth.resize(parameter.GetAreaAreaIndexMaxMag()+1); cellHeight.resize(parameter.GetAreaAreaIndexMaxMag()+1); for (size_t i=0; i<cellWidth.size(); i++) { cellWidth[i]=360.0/pow(2.0,(int)i); } for (size_t i=0; i<cellHeight.size(); i++) { cellHeight[i]=180.0/pow(2.0,(int)i); } // // Writing index file // progress.SetAction("Generating 'areaarea.idx'"); FileWriter writer; FileOffset topLevelOffset=0; FileOffset topLevelOffsetOffset; // Offset of the toplevel entry if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "areaarea.idx"))) { progress.Error("Cannot create 'areaarea.idx'"); return false; } if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "areas.dat"), FileScanner::Sequential, parameter.GetWayDataMemoryMaped())) { progress.Error("Cannot open 'areas.dat'"); return false; } writer.WriteNumber((uint32_t)parameter.GetAreaAreaIndexMaxMag()); // MaxMag if (!writer.GetPos(topLevelOffsetOffset)) { progress.Error("Cannot read current file position"); return false; } if (!writer.WriteFileOffset(topLevelOffset)) { progress.Error("Cannot write top level entry offset"); return false; } int l=parameter.GetAreaAreaIndexMaxMag(); while (l>=0) { size_t areaLevelEntries=0; progress.Info(std::string("Storing level ")+NumberToString(l)+"..."); newAreaLeafs.clear(); SetOffsetOfChildren(leafs,newAreaLeafs); leafs=newAreaLeafs; // Areas if (areas==0 || (areas>0 && areas>areasConsumed)) { uint32_t areaCount=0; progress.Info(std::string("Scanning areas.dat for areas of index level ")+NumberToString(l)+"..."); if (!scanner.GotoBegin()) { progress.Error("Cannot go to begin of way file"); } if (!scanner.Read(areaCount)) { progress.Error("Error while reading number of data entries in file"); return false; } areas=0; for (uint32_t a=1; a<=areaCount; a++) { progress.SetProgress(a,areaCount); FileOffset offset; Area area; scanner.GetPos(offset); if (!area.Read(typeConfig, scanner)) { progress.Error(std::string("Error while reading data entry ")+ NumberToString(a)+" of "+ NumberToString(areaCount)+ " in file '"+ scanner.GetFilename()+"'"); return false; } areas++; double minLon; double maxLon; double minLat; double maxLat; area.GetBoundingBox(minLon,maxLon,minLat,maxLat); // // Calculate highest level where the bounding box completely // fits in the cell size and assign area to the tiles that // hold the geometric center of the tile. // int level=parameter.GetAreaAreaIndexMaxMag(); while (level>=0) { if (maxLon-minLon<=cellWidth[level] && maxLat-minLat<=cellHeight[level]) { break; } level--; } if (level==l) { // // Renormated coordinate space (everything is >=0) // minLon+=180; maxLon+=180; minLat+=90; maxLat+=90; // // Calculate minimum and maximum tile ids that are covered // by the area // uint32_t minyc=(uint32_t)floor(minLat/cellHeight[level]); uint32_t maxyc=(uint32_t)ceil(maxLat/cellHeight[level]); uint32_t minxc=(uint32_t)floor(minLon/cellWidth[level]); uint32_t maxxc=(uint32_t)ceil(maxLon/cellWidth[level]); Entry entry; entry.type=area.GetType()->GetId(); entry.offset=offset; // Add this area to the tile where the center of the area lies in. leafs[Pixel((minxc+maxxc)/2,(minyc+maxyc)/2)].areas.push_back(entry); areaLevelEntries++; areasConsumed++; } } } progress.Debug(std::string("Writing ")+NumberToString(leafs.size())+" leafs ("+ NumberToString(areaLevelEntries)+") "+ "to index of level "+NumberToString(l)+"..."); // Remember the offset of one cell in level '0' if (l==0) { if (!writer.GetPos(topLevelOffset)) { progress.Error("Cannot read top level entry offset"); return false; } } /* uint32_t minX=std::numeric_limits<uint32_t>::max(); uint32_t minY=std::numeric_limits<uint32_t>::max(); uint32_t maxX=std::numeric_limits<uint32_t>::min(); uint32_t maxY=std::numeric_limits<uint32_t>::min(); std::map<TypeId,size_t> useMap; for (std::map<Pixel,AreaLeaf>::const_iterator leaf=leafs.begin(); leaf!=leafs.end(); ++leaf) { minX=std::min(minX,leaf->first.x); maxX=std::max(maxX,leaf->first.x); minY=std::min(minY,leaf->first.y); maxY=std::max(maxY,leaf->first.y); for (std::list<Entry>::const_iterator entry=leaf->second.areas.begin(); entry!=leaf->second.areas.end(); entry++) { std::map<TypeId,size_t>::iterator u=useMap.find(entry->type); if (u==useMap.end()) { useMap[entry->type]=1; } else { u->second++; } } }*/ /* std::cout << "[" << minX << "-" << maxX << "]x[" << minY << "-" << maxY << "] => " << leafs.size() << "/" << (maxX-minX+1)*(maxY-minY+1) << " " << (int)BytesNeededToAddressFileData(leafs.size()) << " " << ByteSizeToString(BytesNeededToAddressFileData(leafs.size())*(maxX-minX+1)*(maxY-minY+1)) << std::endl; for (std::map<TypeId,size_t>::const_iterator u=useMap.begin(); u!=useMap.end(); ++u) { std::cout << "* " << u->first << " " << typeConfig.GetTypeInfo(u->first).GetName() << " " << u->second << std::endl; }*/ if (!WriteIndexLevel(parameter, writer, (int)l, leafs)) { return false; } l--; } writer.SetPos(topLevelOffsetOffset); writer.WriteFileOffset(topLevelOffset); return !writer.HasError() && writer.Close(); }
bool AreaWayIndexGenerator::Import(const ImportParameter& parameter, Progress& progress, const TypeConfig& typeConfig) { FileScanner wayScanner; FileWriter writer; std::set<TypeId> remainingWayTypes; std::vector<TypeData> wayTypeData; size_t level; size_t maxLevel=0; wayTypeData.resize(typeConfig.GetTypes().size()); if (!wayScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "ways.dat"), FileScanner::Sequential, parameter.GetWayDataMemoryMaped())) { progress.Error("Cannot open 'ways.dat'"); return false; } // // Scanning distribution // progress.SetAction("Scanning level distribution of way types"); for (size_t i=0; i<typeConfig.GetTypes().size(); i++) { if (typeConfig.GetTypeInfo(i).CanBeWay() && !typeConfig.GetTypeInfo(i).GetIgnore()) { remainingWayTypes.insert(i); } } level=parameter.GetAreaWayMinMag(); while (!remainingWayTypes.empty()) { uint32_t wayCount=0; std::set<TypeId> currentWayTypes(remainingWayTypes); double cellWidth=360.0/pow(2.0,(int)level); double cellHeight=180.0/pow(2.0,(int)level); std::vector<CoordCountMap> cellFillCount(typeConfig.GetTypes().size()); progress.Info("Scanning Level "+NumberToString(level)+" ("+NumberToString(remainingWayTypes.size())+" types remaining)"); wayScanner.GotoBegin(); if (!wayScanner.Read(wayCount)) { progress.Error("Error while reading number of data entries in file"); return false; } Way way; for (uint32_t w=1; w<=wayCount; w++) { progress.SetProgress(w,wayCount); if (!way.Read(wayScanner)) { progress.Error(std::string("Error while reading data entry ")+ NumberToString(w)+" of "+ NumberToString(wayCount)+ " in file '"+ wayScanner.GetFilename()+"'"); return false; } // Count number of entries per current type and coordinate if (currentWayTypes.find(way.GetType())==currentWayTypes.end()) { continue; } double minLon; double maxLon; double minLat; double maxLat; way.GetBoundingBox(minLon,maxLon,minLat,maxLat); // // Calculate minimum and maximum tile ids that are covered // by the way // Renormated coordinate space (everything is >=0) // uint32_t minxc=(uint32_t)floor((minLon+180.0)/cellWidth); uint32_t maxxc=(uint32_t)floor((maxLon+180.0)/cellWidth); uint32_t minyc=(uint32_t)floor((minLat+90.0)/cellHeight); uint32_t maxyc=(uint32_t)floor((maxLat+90.0)/cellHeight); for (uint32_t y=minyc; y<=maxyc; y++) { for (uint32_t x=minxc; x<=maxxc; x++) { cellFillCount[way.GetType()][Pixel(x,y)]++; } } } // Check if cell fill for current type is in defined limits for (size_t i=0; i<typeConfig.GetTypes().size(); i++) { if (currentWayTypes.find(i)!=currentWayTypes.end()) { CalculateStatistics(level,wayTypeData[i],cellFillCount[i]); if (!FitsIndexCriteria(parameter, progress, typeConfig.GetTypeInfo(i), wayTypeData[i], cellFillCount[i])) { currentWayTypes.erase(i); } } } for (std::set<TypeId>::const_iterator cwt=currentWayTypes.begin(); cwt!=currentWayTypes.end(); cwt++) { maxLevel=std::max(maxLevel,level); progress.Info("Type "+typeConfig.GetTypeInfo(*cwt).GetName()+"(" + NumberToString(*cwt)+"), "+NumberToString(wayTypeData[*cwt].indexCells)+" cells, "+NumberToString(wayTypeData[*cwt].indexEntries)+" objects"); remainingWayTypes.erase(*cwt); } level++; } // // Writing index file // progress.SetAction("Generating 'areaway.idx'"); if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "areaway.idx"))) { progress.Error("Cannot create 'areaway.idx'"); return false; } uint32_t indexEntries=0; for (size_t i=0; i<typeConfig.GetTypes().size(); i++) { if (typeConfig.GetTypeInfo(i).CanBeWay() && wayTypeData[i].HasEntries()) { indexEntries++; } } writer.Write(indexEntries); for (size_t i=0; i<typeConfig.GetTypes().size(); i++) { if (typeConfig.GetTypeInfo(i).CanBeWay() && wayTypeData[i].HasEntries()) { uint8_t dataOffsetBytes=0; FileOffset bitmapOffset=0; writer.WriteNumber(typeConfig.GetTypeInfo(i).GetId()); writer.GetPos(wayTypeData[i].indexOffset); writer.WriteFileOffset(bitmapOffset); if (wayTypeData[i].HasEntries()) { writer.Write(dataOffsetBytes); writer.WriteNumber(wayTypeData[i].indexLevel); writer.WriteNumber(wayTypeData[i].cellXStart); writer.WriteNumber(wayTypeData[i].cellXEnd); writer.WriteNumber(wayTypeData[i].cellYStart); writer.WriteNumber(wayTypeData[i].cellYEnd); } } } for (size_t l=parameter.GetAreaWayMinMag(); l<=maxLevel; l++) { std::set<TypeId> indexTypes; uint32_t wayCount; double cellWidth=360.0/pow(2.0,(int)l); double cellHeight=180.0/pow(2.0,(int)l); for (size_t i=0; i<typeConfig.GetTypes().size(); i++) { if (typeConfig.GetTypeInfo(i).CanBeWay() && wayTypeData[i].HasEntries() && wayTypeData[i].indexLevel==l) { indexTypes.insert(i); } } if (indexTypes.empty()) { continue; } progress.Info("Scanning ways for index level "+NumberToString(l)); std::vector<CoordOffsetsMap> typeCellOffsets(typeConfig.GetTypes().size()); wayScanner.GotoBegin(); if (!wayScanner.Read(wayCount)) { progress.Error("Error while reading number of data entries in file"); return false; } Way way; for (uint32_t w=1; w<=wayCount; w++) { progress.SetProgress(w,wayCount); FileOffset offset; wayScanner.GetPos(offset); if (!way.Read(wayScanner)) { progress.Error(std::string("Error while reading data entry ")+ NumberToString(w)+" of "+ NumberToString(wayCount)+ " in file '"+ wayScanner.GetFilename()+"'"); return false; } if (indexTypes.find(way.GetType())==indexTypes.end()) { continue; } double minLon; double maxLon; double minLat; double maxLat; way.GetBoundingBox(minLon,maxLon,minLat,maxLat); // // Calculate minimum and maximum tile ids that are covered // by the way // Renormated coordinate space (everything is >=0) // uint32_t minxc=(uint32_t)floor((minLon+180.0)/cellWidth); uint32_t maxxc=(uint32_t)floor((maxLon+180.0)/cellWidth); uint32_t minyc=(uint32_t)floor((minLat+90.0)/cellHeight); uint32_t maxyc=(uint32_t)floor((maxLat+90.0)/cellHeight); for (uint32_t y=minyc; y<=maxyc; y++) { for (uint32_t x=minxc; x<=maxxc; x++) { typeCellOffsets[way.GetType()][Pixel(x,y)].push_back(offset); } } } for (std::set<TypeId>::const_iterator type=indexTypes.begin(); type!=indexTypes.end(); ++type) { if (!WriteBitmap(progress, writer, typeConfig.GetTypeInfo(*type), wayTypeData[*type], typeCellOffsets[*type])) { return false; } } } return !writer.HasError() && writer.Close(); }
bool NodeDataGenerator::Import(const TypeConfigRef& typeConfig, const ImportParameter& parameter, Progress& progress) { uint32_t rawNodeCount=0; uint32_t nodesReadCount=0; uint32_t nodesWrittenCount=0; // // Iterator over all raw nodes, hcekc they type, and convert them from raw nodes // to nodes if the type is interesting (!=typeIgnore). // // Count the bounding box by the way... // progress.SetAction("Generating nodes.tmp"); FileScanner scanner; FileWriter writer; if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "rawnodes.dat"), FileScanner::Sequential, parameter.GetRawNodeDataMemoryMaped())) { progress.Error("Cannot open 'rawnodes.dat'"); return false; } if (!scanner.Read(rawNodeCount)) { progress.Error("Error while reading number of data entries in file"); return false; } if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "nodes.tmp"))) { progress.Error("Cannot create 'nodes.tmp'"); return false; } writer.Write(nodesWrittenCount); for (uint32_t n=1; n<=rawNodeCount; n++) { progress.SetProgress(n,rawNodeCount); RawNode rawNode; Node node; if (!rawNode.Read(typeConfig, scanner)) { progress.Error(std::string("Error while reading data entry ")+ NumberToString(n)+" of "+ NumberToString(rawNodeCount)+ " in file '"+ scanner.GetFilename()+"'"); return false; } nodesReadCount++; if (!rawNode.GetType()->GetIgnore()) { node.SetFeatures(rawNode.GetFeatureValueBuffer()); node.SetCoords(rawNode.GetCoords()); FileOffset fileOffset; if (!writer.GetPos(fileOffset)) { progress.Error(std::string("Error while reading current fileOffset in file '")+ writer.GetFilename()+"'"); return false; } writer.Write(rawNode.GetId()); node.Write(typeConfig, writer); nodesWrittenCount++; } } if (!scanner.Close()) { return false; } writer.SetPos(0); writer.Write(nodesWrittenCount); if (!writer.Close()) { return false; } progress.Info(std::string("Read "+NumberToString(nodesReadCount)+" nodes, wrote "+NumberToString(nodesWrittenCount)+" nodes")); return true; }
bool AreaWayIndexGenerator::CalculateDistribution(const TypeConfigRef& typeConfig, const ImportParameter& parameter, Progress& progress, std::vector<TypeData>& wayTypeData, size_t& maxLevel) const { FileScanner wayScanner; TypeInfoSet remainingWayTypes; size_t level; maxLevel=0; wayTypeData.resize(typeConfig->GetTypeCount()); if (!wayScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(), "ways.dat"), FileScanner::Sequential, parameter.GetWayDataMemoryMaped())) { progress.Error("Cannot open 'ways.dat'"); return false; } remainingWayTypes.Set(typeConfig->GetWayTypes()); level=parameter.GetAreaWayMinMag(); while (!remainingWayTypes.Empty()) { uint32_t wayCount=0; TypeInfoSet currentWayTypes(remainingWayTypes); double cellWidth=360.0/pow(2.0,(int)level); double cellHeight=180.0/pow(2.0,(int)level); std::vector<CoordCountMap> cellFillCount(typeConfig->GetTypeCount()); progress.Info("Scanning Level "+NumberToString(level)+" ("+NumberToString(remainingWayTypes.Size())+" types remaining)"); wayScanner.GotoBegin(); if (!wayScanner.Read(wayCount)) { progress.Error("Error while reading number of data entries in file"); return false; } Way way; for (uint32_t w=1; w<=wayCount; w++) { progress.SetProgress(w,wayCount); if (!way.Read(*typeConfig, wayScanner)) { progress.Error(std::string("Error while reading data entry ")+ NumberToString(w)+" of "+ NumberToString(wayCount)+ " in file '"+ wayScanner.GetFilename()+"'"); return false; } // Count number of entries per current type and coordinate if (!currentWayTypes.IsSet(way.GetType())) { continue; } GeoBox boundingBox; way.GetBoundingBox(boundingBox); // // Calculate minimum and maximum tile ids that are covered // by the way // Renormalized coordinate space (everything is >=0) // uint32_t minxc=(uint32_t)floor((boundingBox.GetMinLon()+180.0)/cellWidth); uint32_t maxxc=(uint32_t)floor((boundingBox.GetMaxLon()+180.0)/cellWidth); uint32_t minyc=(uint32_t)floor((boundingBox.GetMinLat()+90.0)/cellHeight); uint32_t maxyc=(uint32_t)floor((boundingBox.GetMaxLat()+90.0)/cellHeight); for (uint32_t y=minyc; y<=maxyc; y++) { for (uint32_t x=minxc; x<=maxxc; x++) { cellFillCount[way.GetType()->GetIndex()][Pixel(x,y)]++; } } } // Check if cell fill for current type is in defined limits for (auto &type : currentWayTypes) { size_t i=type->GetIndex(); CalculateStatistics(level, wayTypeData[i], cellFillCount[i]); if (!FitsIndexCriteria(parameter, progress, *typeConfig->GetTypeInfo(i), wayTypeData[i], cellFillCount[i])) { currentWayTypes.Remove(type); } } for (const auto &type : currentWayTypes) { maxLevel=std::max(maxLevel,level); progress.Info("Type "+type->GetName()+", "+NumberToString(wayTypeData[type->GetIndex()].indexCells)+" cells, "+NumberToString(wayTypeData[type->GetIndex()].indexEntries)+" objects"); remainingWayTypes.Remove(type); } level++; } return !wayScanner.HasError() && wayScanner.Close(); }