int main(int argc,char** argv)
{
NodesX *OSMNodes;
WaysX *OSMWays;
RelationsX *OSMRelations;
ErrorLogsX *OSMErrorLogs;
char *dirname=NULL,*prefix=NULL,*tagging=NULL;
int option_keep=1;
int option_filenames=0;
int arg;
printf_program_start();
/* Parse the command line arguments */
for(arg=1;arg<argc;arg++)
{
if(!strcmp(argv[arg],"--version"))
print_usage(-1,NULL,NULL);
else if(!strcmp(argv[arg],"--help"))
print_usage(1,NULL,NULL);
else if(!strncmp(argv[arg],"--dir=",6))
dirname=&argv[arg][6];
else if(!strncmp(argv[arg],"--sort-ram-size=",16))
option_filesort_ramsize=atoi(&argv[arg][16]);
#if defined(USE_PTHREADS) && USE_PTHREADS
else if(!strncmp(argv[arg],"--sort-threads=",15))
option_filesort_threads=atoi(&argv[arg][15]);
#endif
else if(!strncmp(argv[arg],"--tmpdir=",9))
option_tmpdirname=&argv[arg][9];
else if(!strncmp(argv[arg],"--tagging=",10))
tagging=&argv[arg][10];
else if(!strcmp(argv[arg],"--loggable"))
option_loggable=1;
else if(!strcmp(argv[arg],"--logtime"))
option_logtime=1;
else if(!strcmp(argv[arg],"--logmemory"))
option_logmemory=1;
else if(argv[arg][0]=='-' && argv[arg][1]=='-')
print_usage(0,argv[arg],NULL);
else
option_filenames++;
}
/* Check the specified command line options */
if(!option_filesort_ramsize)
{
#if SLIM
option_filesort_ramsize=64*1024*1024;
#else
option_filesort_ramsize=256*1024*1024;
#endif
}
else
option_filesort_ramsize*=1024*1024;
if(!option_tmpdirname)
{
if(!dirname)
option_tmpdirname=".";
else
option_tmpdirname=dirname;
}
if(tagging)
{
if(!ExistsFile(tagging))
{
fprintf(stderr,"Error: The '--tagging' option specifies a file that does not exist.\n");
exit(EXIT_FAILURE);
}
}
else
{
tagging=FileName(dirname,prefix,"fixme.xml");
if(!ExistsFile(tagging))
{
fprintf(stderr,"Error: The '--tagging' option was not used and the default 'fixme.xml' does not exist.\n");
exit(EXIT_FAILURE);
}
}
if(ParseXMLTaggingRules(tagging))
{
fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging);
exit(EXIT_FAILURE);
}
/* Create new node, segment, way and relation variables */
OSMNodes=NewNodeList(0,0);
OSMWays=NewWayList(0,0);
OSMRelations=NewRelationList(0,0);
/* Create the error log file */
open_errorlog(FileName(dirname,prefix,"fixme.log"),0,option_keep);
/* Parse the file */
for(arg=1;arg<argc;arg++)
{
int fd;
char *filename,*p;
if(argv[arg][0]=='-' && argv[arg][1]=='-')
continue;
filename=strcpy(malloc(strlen(argv[arg])+1),argv[arg]);
fd=OpenFile(filename);
if((p=strstr(filename,".bz2")) && !strcmp(p,".bz2"))
{
fd=Uncompress_Bzip2(fd);
*p=0;
}
if((p=strstr(filename,".gz")) && !strcmp(p,".gz"))
{
fd=Uncompress_Gzip(fd);
*p=0;
}
if((p=strstr(filename,".xz")) && !strcmp(p,".xz"))
{
fd=Uncompress_Xz(fd);
*p=0;
}
printf("\nParse OSM Data [%s]\n==============\n\n",filename);
fflush(stdout);
if((p=strstr(filename,".pbf")) && !strcmp(p,".pbf"))
{
if(ParsePBFFile(fd,OSMNodes,OSMWays,OSMRelations))
exit(EXIT_FAILURE);
}
else if((p=strstr(filename,".o5m")) && !strcmp(p,".o5m"))
{
if(ParseO5MFile(fd,OSMNodes,OSMWays,OSMRelations))
exit(EXIT_FAILURE);
}
else
{
if(ParseOSMFile(fd,OSMNodes,OSMWays,OSMRelations))
exit(EXIT_FAILURE);
}
CloseFile(fd);
free(filename);
}
DeleteXMLTaggingRules();
FinishNodeList(OSMNodes);
FinishWayList(OSMWays);
FinishRelationList(OSMRelations);
/* Sort the data */
printf("\nSort OSM Data\n=============\n\n");
fflush(stdout);
/* Sort the nodes, ways and relations */
SortNodeList(OSMNodes);
SortWayList(OSMWays);
SortRelationList(OSMRelations);
/* Process the data */
RenameFile(OSMNodes->filename_tmp,OSMNodes->filename);
RenameFile(OSMWays->filename_tmp,OSMWays->filename);
RenameFile(OSMRelations->rrfilename_tmp,OSMRelations->rrfilename);
RenameFile(OSMRelations->trfilename_tmp,OSMRelations->trfilename);
close_errorlog();
printf("\nCreate Error Log\n================\n\n");
fflush(stdout);
OSMErrorLogs=NewErrorLogList();
ProcessErrorLogs(OSMErrorLogs,OSMNodes,OSMWays,OSMRelations);
SortErrorLogsGeographically(OSMErrorLogs);
SaveErrorLogs(OSMErrorLogs,FileName(dirname,prefix,"fixme.mem"));
FreeErrorLogList(OSMErrorLogs);
/* Free the memory (delete the temporary files) */
FreeNodeList(OSMNodes,0);
FreeWayList(OSMWays,0);
FreeRelationList(OSMRelations,0);
printf("\n");
fflush(stdout);
printf_program_end();
exit(EXIT_SUCCESS);
}
void PruneIsolatedRegions(NodesX *nodesx,SegmentsX *segmentsx,WaysX *waysx,distance_t minimum)
{
WaysX *newwaysx;
WayX tmpwayx;
transport_t transport;
BitMask *connected,*region;
index_t *regionsegments,*othersegments;
index_t nallocregionsegments,nallocothersegments;
if(nodesx->number==0 || segmentsx->number==0)
return;
/* Map into memory / open the files */
#if !SLIM
nodesx->data=MapFile(nodesx->filename_tmp);
segmentsx->data=MapFileWriteable(segmentsx->filename_tmp);
waysx->data=MapFile(waysx->filename_tmp);
#else
nodesx->fd=SlimMapFile(nodesx->filename_tmp);
segmentsx->fd=SlimMapFileWriteable(segmentsx->filename_tmp);
waysx->fd=SlimMapFile(waysx->filename_tmp);
InvalidateNodeXCache(nodesx->cache);
InvalidateSegmentXCache(segmentsx->cache);
InvalidateWayXCache(waysx->cache);
#endif
newwaysx=NewWayList(0,0);
CloseFileBuffered(newwaysx->fd);
newwaysx->fd=SlimMapFileWriteable(newwaysx->filename_tmp);
connected=AllocBitMask(segmentsx->number);
region =AllocBitMask(segmentsx->number);
logassert(connected,"Failed to allocate memory (try using slim mode?)"); /* Check AllocBitMask() worked */
logassert(region,"Failed to allocate memory (try using slim mode?)"); /* Check AllocBitMask() worked */
regionsegments=(index_t*)malloc((nallocregionsegments=1024)*sizeof(index_t));
othersegments =(index_t*)malloc((nallocothersegments =1024)*sizeof(index_t));
logassert(regionsegments,"Failed to allocate memory (try using slim mode?)"); /* Check malloc() worked */
logassert(othersegments,"Failed to allocate memory (try using slim mode?)"); /* Check malloc() worked */
/* Loop through the transport types */
for(transport=Transport_None+1;transport<Transport_Count;transport++)
{
index_t i,j;
index_t nregions=0,npruned=0,nadjusted=0;
const char *transport_str=TransportName(transport);
transports_t transports=TRANSPORTS(transport);
if(!(waysx->allow&transports))
continue;
/* Print the start message */
printf_first("Pruning Isolated Regions (%s): Segments=0 Adjusted=0 Pruned=0",transport_str);
/* Loop through the segments and find the disconnected ones */
ClearAllBits(connected,segmentsx->number);
ClearAllBits(region ,segmentsx->number);
for(i=0;i<segmentsx->number;i++)
{
index_t nregionsegments=0,nothersegments=0;
distance_t total=0;
SegmentX *segmentx;
WayX *wayx;
if(IsBitSet(connected,i))
goto endloop;
segmentx=LookupSegmentX(segmentsx,i,1);
if(IsPrunedSegmentX(segmentx))
goto endloop;
if(segmentx->way<waysx->number)
wayx=LookupWayX(waysx,segmentx->way,1);
else
SlimFetch(newwaysx->fd,(wayx=&tmpwayx),sizeof(WayX),(segmentx->way-waysx->number)*sizeof(WayX));
if(!(wayx->way.allow&transports))
goto endloop;
othersegments[nothersegments++]=i;
SetBit(region,i);
do
{
index_t thissegment,nodes[2];
thissegment=othersegments[--nothersegments];
if(nregionsegments==nallocregionsegments)
regionsegments=(index_t*)realloc(regionsegments,(nallocregionsegments+=1024)*sizeof(index_t));
regionsegments[nregionsegments++]=thissegment;
segmentx=LookupSegmentX(segmentsx,thissegment,1);
nodes[0]=segmentx->node1;
nodes[1]=segmentx->node2;
total+=DISTANCE(segmentx->distance);
for(j=0;j<2;j++)
{
NodeX *nodex=LookupNodeX(nodesx,nodes[j],1);
if(!(nodex->allow&transports))
continue;
segmentx=FirstSegmentX(segmentsx,nodes[j],1);
while(segmentx)
{
index_t segment=IndexSegmentX(segmentsx,segmentx);
if(segment!=thissegment)
{
if(segmentx->way<waysx->number)
wayx=LookupWayX(waysx,segmentx->way,1);
else
SlimFetch(newwaysx->fd,(wayx=&tmpwayx),sizeof(WayX),(segmentx->way-waysx->number)*sizeof(WayX));
if(wayx->way.allow&transports)
{
/* Already connected - finish */
if(IsBitSet(connected,segment))
{
total=minimum;
goto foundconnection;
}
/* Not in region - add to list */
if(!IsBitSet(region,segment))
{
if(nothersegments==nallocothersegments)
othersegments=(index_t*)realloc(othersegments,(nallocothersegments+=1024)*sizeof(index_t));
othersegments[nothersegments++]=segment;
SetBit(region,segment);
}
}
}
segmentx=NextSegmentX(segmentsx,segmentx,nodes[j]);
}
}
}
while(nothersegments>0 && total<minimum);
foundconnection:
/* Prune the segments or mark them as connected */
if(total<minimum) /* not connected - delete them */
{
nregions++;
for(j=0;j<nregionsegments;j++)
{
SegmentX *segmentx;
WayX *wayx,tmpwayx;
SetBit(connected,regionsegments[j]); /* not really connected, but don't need to check again */
ClearBit(region,regionsegments[j]);
segmentx=LookupSegmentX(segmentsx,regionsegments[j],1);
if(segmentx->way<waysx->number)
wayx=LookupWayX(waysx,segmentx->way,1);
else
SlimFetch(newwaysx->fd,(wayx=&tmpwayx),sizeof(WayX),(segmentx->way-waysx->number)*sizeof(WayX));
if(wayx->way.allow==transports)
{
prune_segment(segmentsx,segmentx);
npruned++;
}
else
{
if(segmentx->way<waysx->number) /* create a new way */
{
tmpwayx=*wayx;
tmpwayx.way.allow&=~transports;
segmentx->way=waysx->number+newwaysx->number;
SlimReplace(newwaysx->fd,&tmpwayx,sizeof(WayX),(segmentx->way-waysx->number)*sizeof(WayX));
newwaysx->number++;
PutBackSegmentX(segmentsx,segmentx);
}
else /* modify the existing one */
{
tmpwayx.way.allow&=~transports;
SlimReplace(newwaysx->fd,&tmpwayx,sizeof(WayX),(segmentx->way-waysx->number)*sizeof(WayX));
}
nadjusted++;
}
}
}
else /* connected - mark as part of the main region */
{
for(j=0;j<nregionsegments;j++)
{
SetBit(connected,regionsegments[j]);
ClearBit(region,regionsegments[j]);
}
for(j=0;j<nothersegments;j++)
{
SetBit(connected,othersegments[j]);
ClearBit(region,othersegments[j]);
}
}
endloop:
if(!((i+1)%10000))
printf_middle("Pruning Isolated Regions (%s): Segments=%"Pindex_t" Adjusted=%"Pindex_t" Pruned=%"Pindex_t" (%"Pindex_t" Regions)",transport_str,i+1,nadjusted,npruned,nregions);
}
/* Print the final message */
printf_last("Pruned Isolated Regions (%s): Segments=%"Pindex_t" Adjusted=%"Pindex_t" Pruned=%"Pindex_t" (%"Pindex_t" Regions)",transport_str,segmentsx->number,nadjusted,npruned,nregions);
}
/* Unmap from memory / close the files */
free(region);
free(connected);
free(regionsegments);
free(othersegments);
#if !SLIM
nodesx->data=UnmapFile(nodesx->data);
segmentsx->data=UnmapFile(segmentsx->data);
waysx->data=UnmapFile(waysx->data);
#else
nodesx->fd=SlimUnmapFile(nodesx->fd);
segmentsx->fd=SlimUnmapFile(segmentsx->fd);
waysx->fd=SlimUnmapFile(waysx->fd);
#endif
SlimUnmapFile(newwaysx->fd);
waysx->number+=newwaysx->number;
waysx->fd=OpenFileBufferedAppend(waysx->filename_tmp);
newwaysx->fd=ReOpenFileBuffered(newwaysx->filename_tmp);
while(!ReadFileBuffered(newwaysx->fd,&tmpwayx,sizeof(WayX)))
WriteFileBuffered(waysx->fd,&tmpwayx,sizeof(WayX));
CloseFileBuffered(waysx->fd);
CloseFileBuffered(newwaysx->fd);
FreeWayList(newwaysx,0);
}
int main(int argc,char** argv)
{
NodesX *Nodes;
SegmentsX *Segments,*SuperSegments=NULL,*MergedSegments=NULL;
WaysX *Ways;
int iteration=0,quit=0;
int max_iterations=10;
char *dirname=NULL,*prefix=NULL,*tagging=NULL;
int option_parse_only=0,option_process_only=0;
int option_filenames=0;
int arg;
/* Parse the command line arguments */
for(arg=1;arg<argc;arg++)
{
if(!strcmp(argv[arg],"--help"))
print_usage(1);
else if(!strcmp(argv[arg],"--slim"))
option_slim=1;
else if(!strncmp(argv[arg],"--sort-ram-size=",16))
option_filesort_ramsize=atoi(&argv[arg][16]);
else if(!strncmp(argv[arg],"--dir=",6))
dirname=&argv[arg][6];
else if(!strncmp(argv[arg],"--tmpdir=",9))
option_tmpdirname=&argv[arg][9];
else if(!strncmp(argv[arg],"--prefix=",9))
prefix=&argv[arg][9];
else if(!strcmp(argv[arg],"--parse-only"))
option_parse_only=1;
else if(!strcmp(argv[arg],"--process-only"))
option_process_only=1;
else if(!strncmp(argv[arg],"--max-iterations=",17))
max_iterations=atoi(&argv[arg][17]);
else if(!strncmp(argv[arg],"--tagging=",10))
tagging=&argv[arg][10];
else if(argv[arg][0]=='-' && argv[arg][1]=='-')
print_usage(0);
else
option_filenames++;
}
/* Check the specified command line options */
if(option_parse_only && option_process_only)
print_usage(0);
if(option_filenames && option_process_only)
print_usage(0);
if(!option_filesort_ramsize)
{
if(option_slim)
option_filesort_ramsize=64*1024*1024;
else
option_filesort_ramsize=256*1024*1024;
}
else
option_filesort_ramsize*=1024*1024;
if(!option_tmpdirname)
{
if(!dirname)
option_tmpdirname=".";
else
option_tmpdirname=dirname;
}
if(tagging && ExistsFile(tagging))
;
else if(!tagging && ExistsFile(FileName(dirname,prefix,"tagging.xml")))
tagging=FileName(dirname,prefix,"tagging.xml");
if(tagging && ParseXMLTaggingRules(tagging))
{
fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging);
return(1);
}
if(!tagging)
{
fprintf(stderr,"Error: Cannot run without reading some tagging rules.\n");
return(1);
}
/* Create new node, segment and way variables */
Nodes=NewNodeList(option_parse_only||option_process_only);
Segments=NewSegmentList(option_parse_only||option_process_only);
Ways=NewWayList(option_parse_only||option_process_only);
/* Parse the file */
if(option_filenames)
{
for(arg=1;arg<argc;arg++)
{
FILE *file;
if(argv[arg][0]=='-' && argv[arg][1]=='-')
continue;
file=fopen(argv[arg],"rb");
if(!file)
{
fprintf(stderr,"Cannot open file '%s' for reading [%s].\n",argv[arg],strerror(errno));
exit(EXIT_FAILURE);
}
printf("\nParse OSM Data [%s]\n==============\n\n",argv[arg]);
fflush(stdout);
if(ParseOSM(file,Nodes,Segments,Ways))
exit(EXIT_FAILURE);
fclose(file);
}
}
else if(!option_process_only)
{
printf("\nParse OSM Data\n==============\n\n");
fflush(stdout);
if(ParseOSM(stdin,Nodes,Segments,Ways))
exit(EXIT_FAILURE);
}
if(option_parse_only)
{
FreeNodeList(Nodes,1);
FreeSegmentList(Segments,1);
FreeWayList(Ways,1);
return(0);
}
/* Process the data */
printf("\nProcess OSM Data\n================\n\n");
fflush(stdout);
/* Sort the nodes, segments and ways */
SortNodeList(Nodes);
SortSegmentList(Segments);
SortWayList(Ways);
/* Remove bad segments (must be after sorting the nodes and segments) */
RemoveBadSegments(Nodes,Segments);
/* Remove non-highway nodes (must be after removing the bad segments) */
RemoveNonHighwayNodes(Nodes,Segments);
/* Measure the segments and replace node/way id with index (must be after removing non-highway nodes) */
UpdateSegments(Segments,Nodes,Ways);
/* Repeated iteration on Super-Nodes and Super-Segments */
do
{
printf("\nProcess Super-Data (iteration %d)\n================================%s\n\n",iteration,iteration>9?"=":"");
fflush(stdout);
if(iteration==0)
{
/* Select the super-nodes */
ChooseSuperNodes(Nodes,Segments,Ways);
/* Select the super-segments */
SuperSegments=CreateSuperSegments(Nodes,Segments,Ways,iteration);
}
else
{
SegmentsX *SuperSegments2;
/* Select the super-nodes */
ChooseSuperNodes(Nodes,SuperSegments,Ways);
/* Select the super-segments */
SuperSegments2=CreateSuperSegments(Nodes,SuperSegments,Ways,iteration);
if(SuperSegments->xnumber==SuperSegments2->xnumber)
quit=1;
FreeSegmentList(SuperSegments,0);
SuperSegments=SuperSegments2;
}
/* Sort the super-segments */
SortSegmentList(SuperSegments);
/* Remove duplicated super-segments */
DeduplicateSegments(SuperSegments,Nodes,Ways);
iteration++;
if(iteration>max_iterations)
quit=1;
}
while(!quit);
/* Combine the super-segments */
printf("\nCombine Segments and Super-Segments\n===================================\n\n");
fflush(stdout);
/* Merge the super-segments */
MergedSegments=MergeSuperSegments(Segments,SuperSegments);
FreeSegmentList(Segments,0);
FreeSegmentList(SuperSegments,0);
Segments=MergedSegments;
/* Rotate segments so that node1<node2 */
RotateSegments(Segments);
/* Sort the segments */
SortSegmentList(Segments);
/* Remove duplicated segments */
DeduplicateSegments(Segments,Nodes,Ways);
/* Cross reference the nodes and segments */
printf("\nCross-Reference Nodes and Segments\n==================================\n\n");
fflush(stdout);
/* Sort the node list geographically */
SortNodeListGeographically(Nodes);
/* Create the real segments and nodes */
CreateRealNodes(Nodes,iteration);
CreateRealSegments(Segments,Ways);
/* Fix the segment and node indexes */
IndexNodes(Nodes,Segments);
IndexSegments(Segments,Nodes);
/* Output the results */
printf("\nWrite Out Database Files\n========================\n\n");
fflush(stdout);
/* Write out the nodes */
SaveNodeList(Nodes,FileName(dirname,prefix,"nodes.mem"));
FreeNodeList(Nodes,0);
/* Write out the segments */
SaveSegmentList(Segments,FileName(dirname,prefix,"segments.mem"));
FreeSegmentList(Segments,0);
/* Write out the ways */
SaveWayList(Ways,FileName(dirname,prefix,"ways.mem"));
FreeWayList(Ways,0);
return(0);
}
