bool RawNode::Read(const TypeConfig& typeConfig,
FileScanner& scanner)
{
if (!scanner.ReadNumber(id)) {
return false;
}
TypeId typeId;
if (!scanner.ReadTypeId(typeId,
typeConfig.GetNodeTypeIdBytes())) {
return false;
}
TypeInfoRef type=typeConfig.GetNodeTypeInfo(typeId);
featureValueBuffer.SetType(type);
if (!type->GetIgnore()) {
if (!featureValueBuffer.Read(scanner)) {
return false;
}
}
if (!scanner.ReadCoord(coords)) {
return false;
}
return !scanner.HasError();
}
bool Way::ReadIds(FileScanner& scanner)
{
ids.resize(nodes.size());
Id minId;
scanner.ReadNumber(minId);
if (minId>0) {
size_t idCurrent=0;
while (idCurrent<ids.size()) {
uint8_t bitset;
size_t bitmask=1;
scanner.Read(bitset);
for (size_t i=0; i<8 && idCurrent<ids.size(); i++) {
if (bitset & bitmask) {
scanner.ReadNumber(ids[idCurrent]);
ids[idCurrent]+=minId;
}
else {
ids[idCurrent]=0;
}
bitmask*=2;
idCurrent++;
}
}
}
return !scanner.HasError();
}
bool Way::ReadOptimized(const TypeConfig& typeConfig,
FileScanner& scanner)
{
if (!scanner.GetPos(fileOffset)) {
return false;
}
TypeId typeId;
scanner.ReadTypeId(typeId,
typeConfig.GetWayTypeIdBytes());
TypeInfoRef type=typeConfig.GetWayTypeInfo(typeId);
featureValueBuffer.SetType(type);
if (!featureValueBuffer.Read(scanner)) {
return false;
}
if (!scanner.Read(nodes)) {
return false;
}
return !scanner.HasError();
}
bool OptimizeAreasLowZoomGenerator::GetAreas(const TypeConfig& typeConfig,
const ImportParameter& parameter,
Progress& progress,
FileScanner& scanner,
const TypeInfoSet& types,
std::vector<std::list<AreaRef> >& areas,
TypeInfoSet& loadedTypes)
{
uint32_t areaCount=0;
size_t collectedAreasCount=0;
loadedTypes=types;
progress.SetAction("Collecting area data to optimize");
scanner.GotoBegin();
scanner.Read(areaCount);
for (uint32_t a=1; a<=areaCount; a++) {
AreaRef area=std::make_shared<Area>();
progress.SetProgress(a,areaCount);
area->Read(typeConfig,
scanner);
if (loadedTypes.IsSet(area->GetType())) {
areas[area->GetType()->GetIndex()].push_back(area);
collectedAreasCount++;
while (collectedAreasCount>parameter.GetOptimizationMaxWayCount() &&
loadedTypes.Size()>1) {
TypeInfoRef victimType;
for (auto &type : loadedTypes) {
if (areas[type->GetIndex()].size()>0 &&
(!victimType ||
areas[type->GetIndex()].size()<areas[victimType->GetIndex()].size())) {
victimType=type;
}
}
assert(victimType);
collectedAreasCount-=areas[victimType->GetIndex()].size();
areas[victimType->GetIndex()].clear();
loadedTypes.Remove(victimType);
}
}
}
progress.Info("Collected "+NumberToString(collectedAreasCount)+" areas for "+NumberToString(loadedTypes.Size())+" types");
return !scanner.HasError();
}
bool Relation::Read(FileScanner& scanner)
{
uint32_t roleCount;
scanner.Read(id);
if (!attributes.Read(scanner)) {
return false;
}
scanner.ReadNumber(roleCount);
if (scanner.HasError()) {
return false;
}
roles.resize(roleCount);
for (size_t i=0; i<roleCount; i++) {
uint32_t nodesCount;
if (!roles[i].attributes.Read(scanner)) {
return false;
}
scanner.Read(roles[i].role);
scanner.ReadNumber(nodesCount);
if (nodesCount>0) {
roles[i].nodes.resize(nodesCount);
Id minId;
uint32_t minLat;
uint32_t minLon;
scanner.Read(minId);
scanner.Read(minLat);
scanner.Read(minLon);
for (size_t j=0; j<nodesCount; j++) {
Id id;
uint32_t latValue;
uint32_t lonValue;
scanner.ReadNumber(id);
scanner.ReadNumber(latValue);
scanner.ReadNumber(lonValue);
roles[i].nodes[j].id=minId+id;
roles[i].nodes[j].lat=(minLat+latValue)/conversionFactor-90.0;
roles[i].nodes[j].lon=(minLon+lonValue)/conversionFactor-180.0;
}
}
}
return !scanner.HasError();
}
/**
* Reads the TurnRestriction data from the given FileScanner
*
* @throws IOException
*/
void TurnRestriction::Read(FileScanner& scanner)
{
uint32_t typeValue;
scanner.ReadNumber(typeValue);
this->type=(Type)typeValue;
scanner.ReadNumber(from);
scanner.ReadNumber(via);
scanner.ReadNumber(to);
}
static void runScan(void)
{
loadMusic();
if ("" != gMusicData->startdir)
{
FileScanner *fscan = new FileScanner();
fscan->SearchDir(gMusicData->startdir);
gMusicData->reloadMusic();
delete fscan;
}
}
static void runScan(void)
{
// maybe we haven't loaded the music yet in which case we wont have a valid music dir set
if (gMusicData->musicDir.isEmpty())
{
QString startdir = gCoreContext->GetSetting("MusicLocation");
startdir = QDir::cleanPath(startdir);
if (!startdir.isEmpty() && !startdir.endsWith("/"))
startdir += "/";
gMusicData->musicDir = startdir;
}
// if we still don't have a valid start dir warn the user and give up
if (gMusicData->musicDir.isEmpty())
{
ShowOkPopup(QObject::tr("You need to tell me where to find your music on the "
"'General Settings' page of MythMusic's settings pages."));
return;
}
if (!QFile::exists(gMusicData->musicDir))
{
ShowOkPopup(QObject::tr("Can't find your music directory. Have you set it correctly on the "
"'General Settings' page of MythMusic's settings pages?"));
return;
}
LOG(VB_GENERAL, LOG_INFO, QString("Scanning '%1' for music files").arg(gMusicData->musicDir));
FileScanner *fscan = new FileScanner();
fscan->SearchDir(gMusicData->musicDir);
// save anything that may have changed
if (gMusicData->all_music && gMusicData->all_music->cleanOutThreads())
gMusicData->all_music->save();
if (gMusicData->all_playlists && gMusicData->all_playlists->cleanOutThreads())
{
gMusicData->all_playlists->save();
int x = gMusicData->all_playlists->getPending();
SavePending(x);
}
// force a complete reload of the tracks and playlists
gPlayer->stop(true);
delete gMusicData;
gMusicData = new MusicData;
loadMusic();
delete fscan;
}
bool LocationIndex::VisitLocationAddressEntries(FileScanner& scanner,
const AdminRegion& region,
const Location& location,
AddressVisitor& visitor,
bool& stopped) const
{
uint32_t addressCount;
if (!scanner.SetPos(location.addressesOffset)) {
return false;
}
if (!scanner.ReadNumber(addressCount)) {
return false;
}
ObjectFileRefStreamReader objectFileRefReader(scanner);
for (size_t i=0; i<addressCount; i++) {
Address address;
if (!scanner.GetPos(address.addressOffset)) {
return false;
}
address.locationOffset=location.locationOffset;
address.regionOffset=location.regionOffset;
if (!scanner.Read(address.name)) {
return false;
}
if (!objectFileRefReader.Read(address.object)) {
return false;
}
if (!visitor.Visit(region,
location,
address)) {
stopped=true;
break;
}
}
return !scanner.HasError();
}
bool LocationIndex::VisitAdminRegions(AdminRegionVisitor& visitor) const
{
FileScanner scanner;
if (!scanner.Open(AppendFileToDir(path,
FILENAME_LOCATION_IDX),
FileScanner::LowMemRandom,
false)) {
log.Error() << "Cannot open file '" << scanner.GetFilename() << "'!";
return false;
}
if (!scanner.SetPos(indexOffset)) {
return false;
}
uint32_t regionCount;
if (!scanner.ReadNumber(regionCount)) {
return false;
}
for (size_t i=0; i<regionCount; i++) {
AdminRegionVisitor::Action action;
FileOffset nextChildOffset;
if (!scanner.ReadFileOffset(nextChildOffset)) {
return false;
}
action=VisitRegionEntries(scanner,
visitor);
if (action==AdminRegionVisitor::error) {
return false;
}
else if (action==AdminRegionVisitor::stop) {
return true;
}
else if (action==AdminRegionVisitor::skipChildren) {
if (i+1<regionCount) {
if (!scanner.SetPos(nextChildOffset)) {
return false;
}
}
}
}
return !scanner.HasError() && scanner.Close();
}
bool LocationIndex::VisitAdminRegionLocations(const AdminRegion& region,
LocationVisitor& visitor,
bool recursive) const
{
FileScanner scanner;
bool stopped=false;
if (!scanner.Open(AppendFileToDir(path,
FILENAME_LOCATION_IDX),
FileScanner::LowMemRandom,
true)) {
log.Error() << "Cannot open file '" << scanner.GetFilename() << "'!";
return false;
}
if (!scanner.SetPos(indexOffset)) {
return false;
}
if (!scanner.SetPos(region.regionOffset)) {
return false;
}
if (!VisitRegionLocationEntries(scanner,
visitor,
recursive,
stopped)) {
return false;
}
return !scanner.HasError() && scanner.Close();
}
bool LocationIndex::ResolveAdminRegionHierachie(const AdminRegionRef& adminRegion,
std::map<FileOffset,AdminRegionRef >& refs) const
{
FileScanner scanner;
if (!scanner.Open(AppendFileToDir(path,
FILENAME_LOCATION_IDX),
FileScanner::LowMemRandom,
true)) {
log.Error() << "Cannot open file '" << scanner.GetFilename() << "'!";
return false;
}
if (!scanner.SetPos(indexOffset)) {
return false;
}
std::list<FileOffset> offsets;
refs[adminRegion->regionOffset]=adminRegion;
if (adminRegion->parentRegionOffset!=0) {
offsets.push_back(adminRegion->parentRegionOffset);
}
while (!offsets.empty()) {
std::list<FileOffset> newOffsets;
for (const auto& offset : offsets) {
if (refs.find(offset)!=refs.end()) {
continue;
}
if (!scanner.SetPos(offset)) {
return false;
}
AdminRegion adminRegion;
if (!LoadAdminRegion(scanner,
adminRegion)) {
return false;
}
refs[adminRegion.regionOffset]=std::make_shared<AdminRegion>(adminRegion);
if (adminRegion.parentRegionOffset!=0) {
newOffsets.push_back(adminRegion.parentRegionOffset);
}
}
offsets.clear();
std::swap(offsets,
newOffsets);
}
return !scanner.HasError() && scanner.Close();
}
static void MusicCallback(void *data, QString &selection)
{
(void) data;
QString sel = selection.toLower();
if (sel == "music_create_playlist")
startDatabaseTree();
else if (sel == "music_play")
startPlayback();
else if (sel == "music_rip")
{
startRipper();
}
else if (sel == "music_import")
{
startImport();
}
else if (sel == "settings_scan")
{
if ("" != gMusicData->startdir)
{
loadMusic();
FileScanner *fscan = new FileScanner();
fscan->SearchDir(gMusicData->startdir);
gMusicData->reloadMusic();
delete fscan;
}
}
else if (sel == "music_set_general")
{
MusicGeneralSettings settings;
settings.exec();
}
else if (sel == "music_set_player")
{
MusicPlayerSettings settings;
settings.exec();
}
else if (sel == "music_set_ripper")
{
MusicRipperSettings settings;
settings.exec();
}
}
bool DebugDatabase::ResolveReferences(const std::string& mapName,
RefType fileType,
const std::set<ObjectOSMRef>& ids,
const std::set<ObjectFileRef>& fileOffsets,
std::map<ObjectOSMRef,ObjectFileRef>& idFileOffsetMap,
std::map<ObjectFileRef,ObjectOSMRef>& fileOffsetIdMap)
{
FileScanner scanner;
uint32_t entryCount;
std::string filename=AppendFileToDir(path,mapName);
if (!scanner.Open(filename,FileScanner::LowMemRandom,false)) {
std::cerr << "Cannot open file '" << scanner.GetFilename() << "'!" << std::endl;
return false;
}
if (!scanner.Read(entryCount)) {
return false;
}
for (size_t i=1; i<=entryCount; i++) {
Id id;
uint8_t typeByte;
OSMRefType osmType;
FileOffset fileOffset;
if (!scanner.Read(id)) {
return false;
}
if (!scanner.Read(typeByte)) {
return false;
}
osmType=(OSMRefType)typeByte;
if (!scanner.ReadFileOffset(fileOffset)) {
return false;
}
ObjectOSMRef osmRef(id,osmType);
ObjectFileRef fileRef(fileOffset,fileType);
if (ids.find(osmRef)!=ids.end() ||
fileOffsets.find(fileRef)!=fileOffsets.end()) {
idFileOffsetMap.insert(std::make_pair(osmRef,fileRef));
fileOffsetIdMap.insert(std::make_pair(fileRef,osmRef));
}
}
return scanner.Close();
}
bool LocationIndex::LoadAdminRegion(FileScanner& scanner,
AdminRegion& region) const
{
uint32_t aliasCount;
if (!scanner.GetPos(region.regionOffset)) {
return false;
}
if (!scanner.ReadFileOffset(region.dataOffset)) {
return false;
}
if (!scanner.ReadFileOffset(region.parentRegionOffset)) {
return false;
}
if (!scanner.Read(region.name)) {
return false;
}
if (!Read(scanner,
region.object)) {
return false;
}
if (!scanner.ReadNumber(aliasCount)) {
return false;
}
region.aliases.clear();
if (aliasCount>0) {
region.aliases.resize(aliasCount);
for (size_t i=0; i<aliasCount; i++) {
if (!scanner.Read(region.aliases[i].name)) {
return false;
}
if (!scanner.ReadFileOffset(region.aliases[i].objectOffset,
bytesForNodeFileOffset)) {
return false;
}
}
}
return !scanner.HasError();
}
bool LocationIndex::Read(FileScanner& scanner,
ObjectFileRef& object) const
{
uint8_t type;
FileOffset fileOffset;
if (!scanner.Read(type)) {
return false;
}
switch (type) {
case refNode:
if (!scanner.ReadFileOffset(fileOffset,
bytesForNodeFileOffset)) {
return false;
}
break;
case refArea:
if (!scanner.ReadFileOffset(fileOffset,
bytesForAreaFileOffset)) {
return false;
}
break;
case refWay:
if (!scanner.ReadFileOffset(fileOffset,
bytesForWayFileOffset)) {
return false;
}
break;
default:
return false;
}
object.Set(fileOffset,
(RefType)type);
return true;
}
/**
* Load the object variant data from the given file.
*
* @param typeConfig
* TypeConfig instance
* @param filename
* Name of the file containing the object variant data
* @return
* True on success, else false
*/
bool ObjectVariantDataFile::Load(const TypeConfig& typeConfig,
const std::string& filename)
{
FileScanner scanner;
data.clear();
isLoaded=false;
this->filename=filename;
try {
scanner.Open(filename,
FileScanner::Sequential,true);
uint32_t dataCount;
scanner.Read(dataCount);
data.resize(dataCount);
for (size_t i=0; i<dataCount; i++) {
data[i].Read(typeConfig,
scanner);
}
scanner.Close();
isLoaded=true;
}
catch (IOException& e) {
log.Error() << e.GetDescription();
scanner.CloseFailsafe();
return false;
}
return true;
}
bool Intersection::Read(FileScanner& scanner)
{
try {
scanner.ReadNumber(nodeId);
uint32_t objectCount;
scanner.ReadNumber(objectCount);
objects.resize(objectCount);
ObjectFileRefStreamReader objectFileRefReader(scanner);
for (size_t i=0; i<objectCount; i++) {
objectFileRefReader.Read(objects[i]);
}
}
catch (IOException& e) {
log.Error() << e.GetDescription();
return false;
}
return true;
}
bool Database::Open(const std::string& path)
{
assert(!path.empty());
this->path=path;
typeConfig=std::make_shared<TypeConfig>();
if (!typeConfig->LoadFromDataFile(path)) {
log.Error() << "Cannot load 'types.dat'!";
return false;
}
FileScanner scanner;
try {
scanner.Open(AppendFileToDir(path,"bounding.dat"),
FileScanner::Normal,
false);
scanner.ReadBox(boundingBox);
log.Debug() << "BoundingBox: " << boundingBox.GetDisplayText();
scanner.Close();
}
catch (IOException& e) {
log.Error() << e.GetDescription();
scanner.CloseFailsafe();
return false;
}
isOpen=true;
return true;
}
bool Node::Read(const TypeConfig& typeConfig,
FileScanner& scanner)
{
if (!scanner.GetPos(fileOffset)) {
return false;
}
uint32_t tmpType;
scanner.ReadNumber(tmpType);
TypeInfoRef type=typeConfig.GetTypeInfo((TypeId)tmpType);
featureValueBuffer.SetType(type);
if (!featureValueBuffer.Read(scanner)) {
return false;
}
scanner.ReadCoord(coords);
return !scanner.HasError();
}
bool Database::Open(const std::string& path)
{
assert(!path.empty());
this->path=path;
typeConfig=new TypeConfig();
if (!typeConfig->LoadFromDataFile(path)) {
log.Error() << "Cannot load 'types.dat'!";
return false;
}
FileScanner scanner;
std::string file=AppendFileToDir(path,"bounding.dat");
if (!scanner.Open(file,FileScanner::Normal,true)) {
log.Error() << "Cannot open '" << scanner.GetFilename() << "'";
return false;
}
if (!scanner.ReadBox(boundingBox)) {
log.Error() << "Error while reading '" << scanner.GetFilename() << "'";
}
log.Debug() << "BoundingBox: " << boundingBox.GetDisplayText();
if (scanner.HasError() || !scanner.Close()) {
log.Error() << "Cannot while reading/closing '" << scanner.GetFilename() << "'";
return false;
}
isOpen=true;
return true;
}
static void loadMusic()
{
// only do this once
if (gMusicData->initialized)
return;
MythScreenStack *popupStack = GetMythMainWindow()->GetStack("popup stack");
QString message = QObject::tr("Loading Music. Please wait ...");
MythUIBusyDialog *busy = new MythUIBusyDialog(message, popupStack,
"musicscanbusydialog");
if (busy->Create())
popupStack->AddScreen(busy, false);
else
busy = NULL;
srand(time(NULL));
CheckFreeDBServerFile();
MSqlQuery count_query(MSqlQuery::InitCon());
bool musicdata_exists = false;
if (count_query.exec("SELECT COUNT(*) FROM music_songs;"))
{
if(count_query.next() &&
0 != count_query.value(0).toInt())
{
musicdata_exists = true;
}
}
// Load all available info about songs (once!)
QString startdir = gCoreContext->GetSetting("MusicLocation");
startdir = QDir::cleanPath(startdir);
if (!startdir.endsWith("/"))
startdir += "/";
Metadata::SetStartdir(startdir);
Decoder::SetLocationFormatUseTags();
// Only search music files if a directory was specified & there
// is no data in the database yet (first run). Otherwise, user
// can choose "Setup" option from the menu to force it.
if (!startdir.isEmpty() && !musicdata_exists)
{
FileScanner *fscan = new FileScanner();
fscan->SearchDir(startdir);
delete fscan;
}
QString paths = gCoreContext->GetSetting("TreeLevels");
// Set the various track formatting modes
Metadata::setArtistAndTrackFormats();
AllMusic *all_music = new AllMusic(paths, startdir);
// Load all playlists into RAM (once!)
PlaylistContainer *all_playlists = new PlaylistContainer(
all_music, gCoreContext->GetHostName());
gMusicData->paths = paths;
gMusicData->startdir = startdir;
gMusicData->all_playlists = all_playlists;
gMusicData->all_music = all_music;
gMusicData->initialized = true;
while (!gMusicData->all_playlists->doneLoading() || !gMusicData->all_music->doneLoading())
{
qApp->processEvents();
usleep(50000);
}
gMusicData->all_playlists->postLoad();
gPlayer->constructPlaylist();
if (busy)
busy->Close();
}
/**
* Reads the data from the given FileScanner
*
* @throws IOException
*/
void RawCoord::Read(const TypeConfig& /*typeConfig*/,
FileScanner& scanner)
{
scanner.ReadNumber(id);
scanner.ReadCoord(coord);
}
bool AreaAreaIndexGenerator::Import(const TypeConfigRef& typeConfig,
const ImportParameter& parameter,
Progress& progress)
{
FileScanner scanner;
size_t areas=0; // Number of areas found
size_t areasConsumed=0; // Number of areas consumed
std::vector<double> cellWidth;
std::vector<double> cellHeight;
std::map<Pixel,AreaLeaf> leafs;
std::map<Pixel,AreaLeaf> newAreaLeafs;
cellWidth.resize(parameter.GetAreaAreaIndexMaxMag()+1);
cellHeight.resize(parameter.GetAreaAreaIndexMaxMag()+1);
for (size_t i=0; i<cellWidth.size(); i++) {
cellWidth[i]=360.0/pow(2.0,(int)i);
}
for (size_t i=0; i<cellHeight.size(); i++) {
cellHeight[i]=180.0/pow(2.0,(int)i);
}
//
// Writing index file
//
progress.SetAction("Generating 'areaarea.idx'");
FileWriter writer;
FileOffset topLevelOffset=0;
FileOffset topLevelOffsetOffset; // Offset of the toplevel entry
if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"areaarea.idx"))) {
progress.Error("Cannot create 'areaarea.idx'");
return false;
}
if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"areas.dat"),
FileScanner::Sequential,
parameter.GetWayDataMemoryMaped())) {
progress.Error("Cannot open 'areas.dat'");
return false;
}
writer.WriteNumber((uint32_t)parameter.GetAreaAreaIndexMaxMag()); // MaxMag
if (!writer.GetPos(topLevelOffsetOffset)) {
progress.Error("Cannot read current file position");
return false;
}
if (!writer.WriteFileOffset(topLevelOffset)) {
progress.Error("Cannot write top level entry offset");
return false;
}
int l=parameter.GetAreaAreaIndexMaxMag();
while (l>=0) {
size_t areaLevelEntries=0;
progress.Info(std::string("Storing level ")+NumberToString(l)+"...");
newAreaLeafs.clear();
SetOffsetOfChildren(leafs,newAreaLeafs);
leafs=newAreaLeafs;
// Areas
if (areas==0 ||
(areas>0 && areas>areasConsumed)) {
uint32_t areaCount=0;
progress.Info(std::string("Scanning areas.dat for areas of index level ")+NumberToString(l)+"...");
if (!scanner.GotoBegin()) {
progress.Error("Cannot go to begin of way file");
}
if (!scanner.Read(areaCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
areas=0;
for (uint32_t a=1; a<=areaCount; a++) {
progress.SetProgress(a,areaCount);
FileOffset offset;
Area area;
scanner.GetPos(offset);
if (!area.Read(typeConfig,
scanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(a)+" of "+
NumberToString(areaCount)+
" in file '"+
scanner.GetFilename()+"'");
return false;
}
areas++;
double minLon;
double maxLon;
double minLat;
double maxLat;
area.GetBoundingBox(minLon,maxLon,minLat,maxLat);
//
// Calculate highest level where the bounding box completely
// fits in the cell size and assign area to the tiles that
// hold the geometric center of the tile.
//
int level=parameter.GetAreaAreaIndexMaxMag();
while (level>=0) {
if (maxLon-minLon<=cellWidth[level] &&
maxLat-minLat<=cellHeight[level]) {
break;
}
level--;
}
if (level==l) {
//
// Renormated coordinate space (everything is >=0)
//
minLon+=180;
maxLon+=180;
minLat+=90;
maxLat+=90;
//
// Calculate minimum and maximum tile ids that are covered
// by the area
//
uint32_t minyc=(uint32_t)floor(minLat/cellHeight[level]);
uint32_t maxyc=(uint32_t)ceil(maxLat/cellHeight[level]);
uint32_t minxc=(uint32_t)floor(minLon/cellWidth[level]);
uint32_t maxxc=(uint32_t)ceil(maxLon/cellWidth[level]);
Entry entry;
entry.type=area.GetType()->GetId();
entry.offset=offset;
// Add this area to the tile where the center of the area lies in.
leafs[Pixel((minxc+maxxc)/2,(minyc+maxyc)/2)].areas.push_back(entry);
areaLevelEntries++;
areasConsumed++;
}
}
}
progress.Debug(std::string("Writing ")+NumberToString(leafs.size())+" leafs ("+
NumberToString(areaLevelEntries)+") "+
"to index of level "+NumberToString(l)+"...");
// Remember the offset of one cell in level '0'
if (l==0) {
if (!writer.GetPos(topLevelOffset)) {
progress.Error("Cannot read top level entry offset");
return false;
}
}
/*
uint32_t minX=std::numeric_limits<uint32_t>::max();
uint32_t minY=std::numeric_limits<uint32_t>::max();
uint32_t maxX=std::numeric_limits<uint32_t>::min();
uint32_t maxY=std::numeric_limits<uint32_t>::min();
std::map<TypeId,size_t> useMap;
for (std::map<Pixel,AreaLeaf>::const_iterator leaf=leafs.begin();
leaf!=leafs.end();
++leaf) {
minX=std::min(minX,leaf->first.x);
maxX=std::max(maxX,leaf->first.x);
minY=std::min(minY,leaf->first.y);
maxY=std::max(maxY,leaf->first.y);
for (std::list<Entry>::const_iterator entry=leaf->second.areas.begin();
entry!=leaf->second.areas.end();
entry++) {
std::map<TypeId,size_t>::iterator u=useMap.find(entry->type);
if (u==useMap.end()) {
useMap[entry->type]=1;
}
else {
u->second++;
}
}
}*/
/*
std::cout << "[" << minX << "-" << maxX << "]x[" << minY << "-" << maxY << "] => " << leafs.size() << "/" << (maxX-minX+1)*(maxY-minY+1) << " " << (int)BytesNeededToAddressFileData(leafs.size()) << " " << ByteSizeToString(BytesNeededToAddressFileData(leafs.size())*(maxX-minX+1)*(maxY-minY+1)) << std::endl;
for (std::map<TypeId,size_t>::const_iterator u=useMap.begin();
u!=useMap.end();
++u) {
std::cout << "* " << u->first << " " << typeConfig.GetTypeInfo(u->first).GetName() << " " << u->second << std::endl;
}*/
if (!WriteIndexLevel(parameter,
writer,
(int)l,
leafs)) {
return false;
}
l--;
}
writer.SetPos(topLevelOffsetOffset);
writer.WriteFileOffset(topLevelOffset);
return !writer.HasError() && writer.Close();
}
/**
* Scans all areas. If an areas is of one of the given merge types, index all node ids
* into the given NodeUseMap for all outer rings of the given area.
*/
bool MergeAreasGenerator::ScanAreaNodeIds(Progress& progress,
const TypeConfig& typeConfig,
FileScanner& scanner,
const TypeInfoSet& mergeTypes,
std::unordered_set<Id>& nodeUseMap)
{
uint32_t areaCount=0;
progress.SetAction("Scanning for nodes joining areas from '"+scanner.GetFilename()+"'");
scanner.GotoBegin();
scanner.Read(areaCount);
uint8_t type;
Id id;
Area data;
std::unordered_set<Id> usedOnceSet;
for (uint32_t current=1; current<=areaCount; current++) {
progress.SetProgress(current,areaCount);
scanner.Read(type);
scanner.Read(id);
data.ReadImport(typeConfig,
scanner);
if (!mergeTypes.IsSet(data.GetType())) {
continue;
}
// We insert every node id only once per area, because we want to
// find nodes that are shared by *different* areas.
std::unordered_set<Id> nodeIds;
for (const auto& ring: data.rings) {
if (!ring.IsOuterRing()) {
continue;
}
for (const auto node : ring.nodes) {
Id id=node.GetId();
if (nodeIds.find(id)==nodeIds.end()) {
auto entry=usedOnceSet.find(id);
if (entry!=usedOnceSet.end()) {
nodeUseMap.insert(id);
}
else {
usedOnceSet.insert(id);
}
nodeIds.insert(id);
}
}
}
}
return true;
}
bool MergeAreasGenerator::Import(const TypeConfigRef& typeConfig,
const ImportParameter& parameter,
Progress& progress)
{
TypeInfoSet mergeTypes;
FileScanner scanner;
FileWriter writer;
uint32_t areasWritten=0;
for (const auto& type : typeConfig->GetTypes()) {
if (type->CanBeArea() &&
type->GetMergeAreas()) {
mergeTypes.Set(type);
}
}
std::unordered_set<Id> nodeUseMap;
try {
scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
MergeAreaDataGenerator::AREAS_TMP),
FileScanner::Sequential,
parameter.GetRawWayDataMemoryMaped());
if (!ScanAreaNodeIds(progress,
*typeConfig,
scanner,
mergeTypes,
nodeUseMap)) {
return false;
}
uint32_t nodeCount=nodeUseMap.size();
progress.Info("Found "+NumberToString(nodeCount)+" nodes as possible connection points for areas");
/* ------ */
writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
AREAS2_TMP));
writer.Write(areasWritten);
while (true) {
TypeInfoSet loadedTypes;
std::vector<AreaMergeData> mergeJob(typeConfig->GetTypeCount());
//
// Load type data
//
progress.SetAction("Collecting area data by type");
if (!GetAreas(parameter,
progress,
*typeConfig,
mergeTypes,
loadedTypes,
nodeUseMap,
scanner,
writer,
mergeJob,
areasWritten)) {
return false;
}
// Merge
progress.SetAction("Merging areas");
for (const auto& type : loadedTypes) {
if (!mergeJob[type->GetIndex()].areas.empty()) {
progress.Info("Merging areas of type "+type->GetName());
MergeAreas(progress,
nodeUseMap,
mergeJob[type->GetIndex()]);
progress.Info("Reduced areas of '"+type->GetName()+"' from "+NumberToString(mergeJob[type->GetIndex()].areaCount)+" to "+NumberToString(mergeJob[type->GetIndex()].areaCount-mergeJob[type->GetIndex()].mergedAway.size()));
mergeJob[type->GetIndex()].areas.clear();
}
}
// Store back merge result
if (!loadedTypes.Empty()) {
if (!WriteMergeResult(progress,
*typeConfig,
scanner,
writer,
loadedTypes,
mergeJob,
areasWritten)) {
return false;
}
mergeTypes.Remove(loadedTypes);
}
if (mergeTypes.Empty()) {
break;
}
}
scanner.Close();
writer.GotoBegin();
writer.Write(areasWritten);
writer.Close();
}
catch (IOException& e) {
progress.Error(e.GetDescription());
scanner.CloseFailsafe();
writer.CloseFailsafe();
return false;
}
return true;
}
bool MergeAreasGenerator::WriteMergeResult(Progress& progress,
const TypeConfig& typeConfig,
FileScanner& scanner,
FileWriter& writer,
const TypeInfoSet& loadedTypes,
std::vector<AreaMergeData>& mergeJob,
uint32_t& areasWritten)
{
uint32_t areaCount=0;
std::unordered_map<FileOffset,AreaRef> merges;
std::unordered_set<FileOffset> ignores;
for (const auto& type : loadedTypes) {
for (const auto& area : mergeJob[type->GetIndex()].merges) {
merges[area->GetFileOffset()]=area;
}
ignores.insert(mergeJob[type->GetIndex()].mergedAway.begin(),
mergeJob[type->GetIndex()].mergedAway.end());
}
scanner.GotoBegin();
scanner.Read(areaCount);
for (uint32_t a=1; a<=areaCount; a++) {
uint8_t type;
Id id;
AreaRef area=std::make_shared<Area>();
progress.SetProgress(a,areaCount);
scanner.Read(type);
scanner.Read(id);
area->ReadImport(typeConfig,
scanner);
if (loadedTypes.IsSet(area->GetType())) {
if (ignores.find(area->GetFileOffset())!=ignores.end()) {
continue;
}
writer.Write(type);
writer.Write(id);
const auto& merge=merges.find(area->GetFileOffset()) ;
if (merge!=merges.end()) {
area=merge->second;
}
area->WriteImport(typeConfig,
writer);
areasWritten++;
}
}
return true;
}
/**
* Load areas which has a one for the types given by types. If at leats one node
* in one of the outer rings of the areas is marked in nodeUseMap as "used at least twice",
* index it into the areas map.
*
* If the number of indexed areas is bigger than parameter.GetRawWayBlockSize() types are
* dropped form areas until the number is again below the lmit.
*/
bool MergeAreasGenerator::GetAreas(const ImportParameter& parameter,
Progress& progress,
const TypeConfig& typeConfig,
const TypeInfoSet& candidateTypes,
TypeInfoSet& loadedTypes,
const std::unordered_set<Id>& nodeUseMap,
FileScanner& scanner,
FileWriter& writer,
std::vector<AreaMergeData>& mergeJob,
uint32_t& areasWritten)
{
bool firstCall=areasWritten==0; // We are called for the first time
uint32_t areaCount=0;
size_t collectedAreasCount=0;
size_t typesWithAreas=0;
for (auto& data : mergeJob) {
data.areaCount=0;
}
loadedTypes=candidateTypes;
scanner.GotoBegin();
scanner.Read(areaCount);
for (uint32_t a=1; a<=areaCount; a++) {
uint8_t type;
Id id;
AreaRef area=std::make_shared<Area>();
progress.SetProgress(a,areaCount);
scanner.Read(type);
scanner.Read(id);
area->ReadImport(typeConfig,
scanner);
mergeJob[area->GetType()->GetIndex()].areaCount++;
// This is an area of a type that does not get merged,
// we directly store it in the target file.
if (!loadedTypes.IsSet(area->GetType())) {
if (firstCall) {
writer.Write(type);
writer.Write(id);
area->WriteImport(typeConfig,
writer);
areasWritten++;
}
continue;
}
bool isMergeCandidate=false;
for (const auto& ring: area->rings) {
if (!ring.IsOuterRing()) {
continue;
}
for (const auto node : ring.nodes) {
if (nodeUseMap.find(node.GetId())!=nodeUseMap.end()) {
isMergeCandidate=true;
break;
}
}
if (isMergeCandidate) {
break;
}
}
if (!isMergeCandidate) {
continue;
}
if (mergeJob[area->GetType()->GetIndex()].areas.empty()) {
typesWithAreas++;
}
mergeJob[area->GetType()->GetIndex()].areas.push_back(area);
collectedAreasCount++;
while (collectedAreasCount>parameter.GetRawWayBlockSize() &&
typesWithAreas>1) {
TypeInfoRef victimType;
// Find the type with the smallest amount of ways loaded
for (auto &type : loadedTypes) {
if (!mergeJob[type->GetIndex()].areas.empty() &&
(!victimType ||
mergeJob[type->GetIndex()].areas.size()<mergeJob[victimType->GetIndex()].areas.size())) {
victimType=type;
}
}
// If there is more then one type of way, we always must find a "victim" type.
assert(victimType);
// Correct the statistics
collectedAreasCount-=mergeJob[victimType->GetIndex()].areas.size();
// Clear already loaded data of th victim type
mergeJob[victimType->GetIndex()].areas.clear();
typesWithAreas--;
loadedTypes.Remove(victimType);
}
}
progress.SetAction("Collected "+NumberToString(collectedAreasCount)+" areas for "+NumberToString(loadedTypes.Size())+" types");
return true;
}
bool NodeDataGenerator::Import(const TypeConfigRef& typeConfig,
const ImportParameter& parameter,
Progress& progress)
{
uint32_t rawNodeCount=0;
uint32_t nodesReadCount=0;
uint32_t nodesWrittenCount=0;
//
// Iterator over all raw nodes, hcekc they type, and convert them from raw nodes
// to nodes if the type is interesting (!=typeIgnore).
//
// Count the bounding box by the way...
//
progress.SetAction("Generating nodes.tmp");
FileScanner scanner;
FileWriter writer;
if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"rawnodes.dat"),
FileScanner::Sequential,
parameter.GetRawNodeDataMemoryMaped())) {
progress.Error("Cannot open 'rawnodes.dat'");
return false;
}
if (!scanner.Read(rawNodeCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
if (!writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"nodes.tmp"))) {
progress.Error("Cannot create 'nodes.tmp'");
return false;
}
writer.Write(nodesWrittenCount);
for (uint32_t n=1; n<=rawNodeCount; n++) {
progress.SetProgress(n,rawNodeCount);
RawNode rawNode;
Node node;
if (!rawNode.Read(typeConfig,
scanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(n)+" of "+
NumberToString(rawNodeCount)+
" in file '"+
scanner.GetFilename()+"'");
return false;
}
nodesReadCount++;
if (!rawNode.GetType()->GetIgnore()) {
node.SetFeatures(rawNode.GetFeatureValueBuffer());
node.SetCoords(rawNode.GetCoords());
FileOffset fileOffset;
if (!writer.GetPos(fileOffset)) {
progress.Error(std::string("Error while reading current fileOffset in file '")+
writer.GetFilename()+"'");
return false;
}
writer.Write(rawNode.GetId());
node.Write(typeConfig,
writer);
nodesWrittenCount++;
}
}
if (!scanner.Close()) {
return false;
}
writer.SetPos(0);
writer.Write(nodesWrittenCount);
if (!writer.Close()) {
return false;
}
progress.Info(std::string("Read "+NumberToString(nodesReadCount)+" nodes, wrote "+NumberToString(nodesWrittenCount)+" nodes"));
return true;
}
bool OptimizeAreaWayIdsGenerator::ScanAreaIds(const ImportParameter& parameter,
Progress& progress,
const TypeConfig& typeConfig,
NodeUseMap& nodeUseMap)
{
FileScanner scanner;
uint32_t dataCount=0;
progress.SetAction("Scanning ids from 'areas2.tmp'");
if (!scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
"areas2.tmp"),
FileScanner::Sequential,
parameter.GetAreaDataMemoryMaped())) {
progress.Error(std::string("Cannot open '")+scanner.GetFilename()+"'");
return false;
}
if (!scanner.Read(dataCount)) {
progress.Error("Error while reading number of data entries in file");
return false;
}
for (uint32_t current=1; current<=dataCount; current++) {
uint8_t type;
Id id;
Area data;
progress.SetProgress(current,dataCount);
if (!scanner.Read(type) ||
!scanner.Read(id) ||
!data.ReadImport(typeConfig,
scanner)) {
progress.Error(std::string("Error while reading data entry ")+
NumberToString(current)+" of "+
NumberToString(dataCount)+
" in file '"+
scanner.GetFilename()+"'");
return false;
}
for (const auto& ring: data.rings) {
std::unordered_set<Id> nodeIds;
if (!ring.GetType()->CanRoute()) {
continue;
}
for (const auto id: ring.ids) {
if (nodeIds.find(id)==nodeIds.end()) {
nodeUseMap.SetNodeUsed(id);
nodeIds.insert(id);
}
}
}
}
if (!scanner.Close()) {
progress.Error(std::string("Error while closing file '")+
scanner.GetFilename()+"'");
return false;
}
return true;
}