Preprocess::Callback::Callback(const TypeConfigRef& typeConfig,
const ImportParameter& parameter,
Progress& progress)
: typeConfig(typeConfig),
parameter(parameter),
progress(progress),
coordCount(0),
nodeCount(0),
wayCount(0),
areaCount(0),
relationCount(0),
coastlineCount(0),
turnRestrictionCount(0),
multipolygonCount(0),
lastNodeId(std::numeric_limits<OSMId>::min()),
lastWayId(std::numeric_limits<OSMId>::min()),
lastRelationId(std::numeric_limits<OSMId>::min()),
nodeSortingError(false),
waySortingError(false),
relationSortingError(false),
coordPageCount(0),
currentPageId(std::numeric_limits<PageId>::max()),
currentPageOffset(0)
{
minCoord.Set(90.0,180.0);
maxCoord.Set(-90.0,-180.0);
nodeStat.resize(typeConfig->GetTypeCount(),0);
areaStat.resize(typeConfig->GetTypeCount(),0);
wayStat.resize(typeConfig->GetTypeCount(),0);
}
ImportErrorReporter::ImportErrorReporter(Progress& progress,
const TypeConfigRef& typeConfig,
const std::string& destinationDirectory)
: progress(progress),
typeConfig(typeConfig),
destinationDirectory(destinationDirectory),
wayErrorCount(0),
relationErrorCount(0),
locationErrorCount(0)
{
nameTagId=typeConfig->GetTagId("name");
wayReport.Open(AppendFileToDir(destinationDirectory,FILENAME_WAY_HTML));
wayReport.WriteDocumentStart();
wayReport.WriteHeader("Way format errors","Import errors related to ways","","..stylesheet.css");
wayReport.WriteBodyStart();
wayReport.WriteListStart();
relationReport.Open(AppendFileToDir(destinationDirectory,FILENAME_RELATION_HTML));
relationReport.WriteDocumentStart();
relationReport.WriteHeader("Relation format errors","Import errors related to relations","","..stylesheet.css");
relationReport.WriteBodyStart();
relationReport.WriteListStart();
locationReport.Open(AppendFileToDir(destinationDirectory,FILENAME_LOCATION_HTML));
locationReport.WriteDocumentStart();
locationReport.WriteHeader("Location format errors","Import errors related to location information","","..stylesheet.css");
locationReport.WriteBodyStart();
locationReport.WriteListStart();
index.Open(AppendFileToDir(destinationDirectory,FILENAME_INDEX_HTML));
index.WriteDocumentStart();
index.WriteHeader("Index","Index of all reports","","..stylesheet.css");
index.WriteBodyStart();
index.WriteListStart();
index.WriteListEntryStart();
index.WriteLink("way.html","List of way import errors");
index.WriteListEntryEnd();
index.WriteListEntryStart();
index.WriteLink("relation.html","List of relation import errors");
index.WriteListEntryEnd();
index.WriteListEntryStart();
index.WriteLink("location.html","List of location related import errors");
index.WriteListEntryEnd();
index.WriteListEnd();
index.WriteBodyEnd();
index.WriteDocumentEnd();
index.CloseFailsafe();
}
bool AreaWayIndex::Load(const TypeConfigRef& typeConfig,
const std::string& path)
{
datafilename=path+"/"+filepart;
if (!scanner.Open(datafilename,FileScanner::LowMemRandom,true)) {
log.Error() << "Cannot open file '" << scanner.GetFilename() << "'";
return false;
}
uint32_t indexEntries;
scanner.Read(indexEntries);
for (size_t i=0; i<indexEntries; i++) {
TypeId type;
scanner.ReadTypeId(type,
typeConfig->GetWayTypeIdBytes());
if (type>=wayTypeData.size()) {
wayTypeData.resize(type+1);
}
scanner.ReadFileOffset(wayTypeData[type].bitmapOffset);
if (wayTypeData[type].bitmapOffset>0) {
scanner.Read(wayTypeData[type].dataOffsetBytes);
scanner.ReadNumber(wayTypeData[type].indexLevel);
scanner.ReadNumber(wayTypeData[type].cellXStart);
scanner.ReadNumber(wayTypeData[type].cellXEnd);
scanner.ReadNumber(wayTypeData[type].cellYStart);
scanner.ReadNumber(wayTypeData[type].cellYEnd);
wayTypeData[type].cellXCount=wayTypeData[type].cellXEnd-wayTypeData[type].cellXStart+1;
wayTypeData[type].cellYCount=wayTypeData[type].cellYEnd-wayTypeData[type].cellYStart+1;
wayTypeData[type].cellWidth=360.0/pow(2.0,(int)wayTypeData[type].indexLevel);
wayTypeData[type].cellHeight=180.0/pow(2.0,(int)wayTypeData[type].indexLevel);
wayTypeData[type].minLon=wayTypeData[type].cellXStart*wayTypeData[type].cellWidth-180.0;
wayTypeData[type].maxLon=(wayTypeData[type].cellXEnd+1)*wayTypeData[type].cellWidth-180.0;
wayTypeData[type].minLat=wayTypeData[type].cellYStart*wayTypeData[type].cellHeight-90.0;
wayTypeData[type].maxLat=(wayTypeData[type].cellYEnd+1)*wayTypeData[type].cellHeight-90.0;
}
}
return !scanner.HasError() && scanner.Close();
}
bool MergeAreasGenerator::Import(const TypeConfigRef& typeConfig,
const ImportParameter& parameter,
Progress& progress)
{
TypeInfoSet mergeTypes;
FileScanner scanner;
FileWriter writer;
uint32_t areasWritten=0;
for (const auto& type : typeConfig->GetTypes()) {
if (type->CanBeArea() &&
type->GetMergeAreas()) {
mergeTypes.Set(type);
}
}
std::unordered_set<Id> nodeUseMap;
try {
scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
MergeAreaDataGenerator::AREAS_TMP),
FileScanner::Sequential,
parameter.GetRawWayDataMemoryMaped());
if (!ScanAreaNodeIds(progress,
*typeConfig,
scanner,
mergeTypes,
nodeUseMap)) {
return false;
}
uint32_t nodeCount=nodeUseMap.size();
progress.Info("Found "+NumberToString(nodeCount)+" nodes as possible connection points for areas");
/* ------ */
writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
AREAS2_TMP));
writer.Write(areasWritten);
while (true) {
TypeInfoSet loadedTypes;
std::vector<AreaMergeData> mergeJob(typeConfig->GetTypeCount());
//
// Load type data
//
progress.SetAction("Collecting area data by type");
if (!GetAreas(parameter,
progress,
*typeConfig,
mergeTypes,
loadedTypes,
nodeUseMap,
scanner,
writer,
mergeJob,
areasWritten)) {
return false;
}
// Merge
progress.SetAction("Merging areas");
for (const auto& type : loadedTypes) {
if (!mergeJob[type->GetIndex()].areas.empty()) {
progress.Info("Merging areas of type "+type->GetName());
MergeAreas(progress,
nodeUseMap,
mergeJob[type->GetIndex()]);
progress.Info("Reduced areas of '"+type->GetName()+"' from "+NumberToString(mergeJob[type->GetIndex()].areaCount)+" to "+NumberToString(mergeJob[type->GetIndex()].areaCount-mergeJob[type->GetIndex()].mergedAway.size()));
mergeJob[type->GetIndex()].areas.clear();
}
}
// Store back merge result
if (!loadedTypes.Empty()) {
if (!WriteMergeResult(progress,
*typeConfig,
scanner,
writer,
loadedTypes,
mergeJob,
areasWritten)) {
return false;
}
mergeTypes.Remove(loadedTypes);
}
if (mergeTypes.Empty()) {
break;
}
}
scanner.Close();
writer.GotoBegin();
writer.Write(areasWritten);
writer.Close();
}
catch (IOException& e) {
progress.Error(e.GetDescription());
scanner.CloseFailsafe();
writer.CloseFailsafe();
return false;
}
return true;
}
bool AreaWayIndexGenerator::Import(const TypeConfigRef& typeConfig,
const ImportParameter& parameter,
Progress& progress)
{
FileScanner wayScanner;
FileWriter writer;
std::vector<TypeData> wayTypeData;
size_t maxLevel;
progress.Info("Minimum magnification: "+NumberToString(parameter.GetAreaWayMinMag()));
//
// Scanning distribution
//
progress.SetAction("Scanning level distribution of way types");
if (!CalculateDistribution(typeConfig,
parameter,
progress,
wayTypeData,
maxLevel)) {
return false;
}
// Calculate number of types which have data
uint32_t indexEntries=0;
for (const auto& type : typeConfig->GetWayTypes())
{
if (wayTypeData[type->GetIndex()].HasEntries()) {
indexEntries++;
}
}
//
// Writing index file
//
progress.SetAction("Generating 'areaway.idx'");
if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"areaway.idx"))) {
progress.Error("Cannot create 'areaway.idx'");
return false;
}
writer.Write(indexEntries);
for (const auto &type : typeConfig->GetWayTypes()) {
size_t i=type->GetIndex();
if (wayTypeData[i].HasEntries()) {
uint8_t dataOffsetBytes=0;
FileOffset bitmapOffset=0;
writer.WriteTypeId(type->GetWayId(),
typeConfig->GetWayTypeIdBytes());
writer.GetPos(wayTypeData[i].indexOffset);
writer.WriteFileOffset(bitmapOffset);
writer.Write(dataOffsetBytes);
writer.WriteNumber(wayTypeData[i].indexLevel);
writer.WriteNumber(wayTypeData[i].cellXStart);
writer.WriteNumber(wayTypeData[i].cellXEnd);
writer.WriteNumber(wayTypeData[i].cellYStart);
writer.WriteNumber(wayTypeData[i].cellYEnd);
}
}
if (!wayScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"ways.dat"),
FileScanner::Sequential,
parameter.GetWayDataMemoryMaped())) {
progress.Error("Cannot open 'ways.dat'");
return false;
}
for (size_t l=parameter.GetAreaWayMinMag(); l<=maxLevel; l++) {
TypeInfoSet indexTypes(*typeConfig);
uint32_t wayCount;
double cellWidth=360.0/pow(2.0,(int)l);
double cellHeight=180.0/pow(2.0,(int)l);
wayScanner.GotoBegin();
for (const auto &type : typeConfig->GetWayTypes()) {
if (wayTypeData[type->GetIndex()].HasEntries() &&
wayTypeData[type->GetIndex()].indexLevel==l) {
indexTypes.Set(type);
}
}
if (indexTypes.Empty()) {
continue;
}
progress.Info("Scanning ways for index level "+NumberToString(l));
std::vector<CoordOffsetsMap> typeCellOffsets(typeConfig->GetTypeCount());
if (!wayScanner.Read(wayCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
Way way;
for (uint32_t w=1; w<=wayCount; w++) {
progress.SetProgress(w,wayCount);
FileOffset offset;
wayScanner.GetPos(offset);
if (!way.Read(*typeConfig,
wayScanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(w)+" of "+
NumberToString(wayCount)+
" in file '"+
wayScanner.GetFilename()+"'");
return false;
}
if (!indexTypes.IsSet(way.GetType())) {
continue;
}
GeoBox boundingBox;
way.GetBoundingBox(boundingBox);
//
// Calculate minimum and maximum tile ids that are covered
// by the way
// Renormalized coordinate space (everything is >=0)
//
uint32_t minxc=(uint32_t)floor((boundingBox.GetMinLon()+180.0)/cellWidth);
uint32_t maxxc=(uint32_t)floor((boundingBox.GetMaxLon()+180.0)/cellWidth);
uint32_t minyc=(uint32_t)floor((boundingBox.GetMinLat()+90.0)/cellHeight);
uint32_t maxyc=(uint32_t)floor((boundingBox.GetMaxLat()+90.0)/cellHeight);
for (uint32_t y=minyc; y<=maxyc; y++) {
for (uint32_t x=minxc; x<=maxxc; x++) {
typeCellOffsets[way.GetType()->GetIndex()][Pixel(x,y)].push_back(offset);
}
}
}
for (const auto &type : indexTypes) {
size_t index=type->GetIndex();
if (!WriteBitmap(progress,
writer,
*typeConfig->GetTypeInfo(index),
wayTypeData[index],
typeCellOffsets[index])) {
return false;
}
}
}
return !writer.HasError() && writer.Close();
}
bool AreaWayIndexGenerator::CalculateDistribution(const TypeConfigRef& typeConfig,
const ImportParameter& parameter,
Progress& progress,
std::vector<TypeData>& wayTypeData,
size_t& maxLevel) const
{
FileScanner wayScanner;
TypeInfoSet remainingWayTypes;
size_t level;
maxLevel=0;
wayTypeData.resize(typeConfig->GetTypeCount());
if (!wayScanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"ways.dat"),
FileScanner::Sequential,
parameter.GetWayDataMemoryMaped())) {
progress.Error("Cannot open 'ways.dat'");
return false;
}
remainingWayTypes.Set(typeConfig->GetWayTypes());
level=parameter.GetAreaWayMinMag();
while (!remainingWayTypes.Empty()) {
uint32_t wayCount=0;
TypeInfoSet currentWayTypes(remainingWayTypes);
double cellWidth=360.0/pow(2.0,(int)level);
double cellHeight=180.0/pow(2.0,(int)level);
std::vector<CoordCountMap> cellFillCount(typeConfig->GetTypeCount());
progress.Info("Scanning Level "+NumberToString(level)+" ("+NumberToString(remainingWayTypes.Size())+" types remaining)");
wayScanner.GotoBegin();
if (!wayScanner.Read(wayCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
Way way;
for (uint32_t w=1; w<=wayCount; w++) {
progress.SetProgress(w,wayCount);
if (!way.Read(*typeConfig,
wayScanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(w)+" of "+
NumberToString(wayCount)+
" in file '"+
wayScanner.GetFilename()+"'");
return false;
}
// Count number of entries per current type and coordinate
if (!currentWayTypes.IsSet(way.GetType())) {
continue;
}
GeoBox boundingBox;
way.GetBoundingBox(boundingBox);
//
// Calculate minimum and maximum tile ids that are covered
// by the way
// Renormalized coordinate space (everything is >=0)
//
uint32_t minxc=(uint32_t)floor((boundingBox.GetMinLon()+180.0)/cellWidth);
uint32_t maxxc=(uint32_t)floor((boundingBox.GetMaxLon()+180.0)/cellWidth);
uint32_t minyc=(uint32_t)floor((boundingBox.GetMinLat()+90.0)/cellHeight);
uint32_t maxyc=(uint32_t)floor((boundingBox.GetMaxLat()+90.0)/cellHeight);
for (uint32_t y=minyc; y<=maxyc; y++) {
for (uint32_t x=minxc; x<=maxxc; x++) {
cellFillCount[way.GetType()->GetIndex()][Pixel(x,y)]++;
}
}
}
// Check if cell fill for current type is in defined limits
for (auto &type : currentWayTypes) {
size_t i=type->GetIndex();
CalculateStatistics(level,
wayTypeData[i],
cellFillCount[i]);
if (!FitsIndexCriteria(parameter,
progress,
*typeConfig->GetTypeInfo(i),
wayTypeData[i],
cellFillCount[i])) {
currentWayTypes.Remove(type);
}
}
for (const auto &type : currentWayTypes) {
maxLevel=std::max(maxLevel,level);
progress.Info("Type "+type->GetName()+", "+NumberToString(wayTypeData[type->GetIndex()].indexCells)+" cells, "+NumberToString(wayTypeData[type->GetIndex()].indexEntries)+" objects");
remainingWayTypes.Remove(type);
}
level++;
}
return !wayScanner.HasError() && wayScanner.Close();
}