int main(int argc,char** argv) { NodesX *OSMNodes; WaysX *OSMWays; RelationsX *OSMRelations; ErrorLogsX *OSMErrorLogs; char *dirname=NULL,*prefix=NULL,*tagging=NULL; int option_keep=1; int option_filenames=0; int arg; printf_program_start(); /* Parse the command line arguments */ for(arg=1;arg<argc;arg++) { if(!strcmp(argv[arg],"--version")) print_usage(-1,NULL,NULL); else if(!strcmp(argv[arg],"--help")) print_usage(1,NULL,NULL); else if(!strncmp(argv[arg],"--dir=",6)) dirname=&argv[arg][6]; else if(!strncmp(argv[arg],"--sort-ram-size=",16)) option_filesort_ramsize=atoi(&argv[arg][16]); #if defined(USE_PTHREADS) && USE_PTHREADS else if(!strncmp(argv[arg],"--sort-threads=",15)) option_filesort_threads=atoi(&argv[arg][15]); #endif else if(!strncmp(argv[arg],"--tmpdir=",9)) option_tmpdirname=&argv[arg][9]; else if(!strncmp(argv[arg],"--tagging=",10)) tagging=&argv[arg][10]; else if(!strcmp(argv[arg],"--loggable")) option_loggable=1; else if(!strcmp(argv[arg],"--logtime")) option_logtime=1; else if(!strcmp(argv[arg],"--logmemory")) option_logmemory=1; else if(argv[arg][0]=='-' && argv[arg][1]=='-') print_usage(0,argv[arg],NULL); else option_filenames++; } /* Check the specified command line options */ if(!option_filesort_ramsize) { #if SLIM option_filesort_ramsize=64*1024*1024; #else option_filesort_ramsize=256*1024*1024; #endif } else option_filesort_ramsize*=1024*1024; if(!option_tmpdirname) { if(!dirname) option_tmpdirname="."; else option_tmpdirname=dirname; } if(tagging) { if(!ExistsFile(tagging)) { fprintf(stderr,"Error: The '--tagging' option specifies a file that does not exist.\n"); exit(EXIT_FAILURE); } } else { tagging=FileName(dirname,prefix,"fixme.xml"); if(!ExistsFile(tagging)) { fprintf(stderr,"Error: The '--tagging' option was not used and the default 'fixme.xml' does not exist.\n"); exit(EXIT_FAILURE); } } if(ParseXMLTaggingRules(tagging)) { fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging); exit(EXIT_FAILURE); } /* Create new node, segment, way and relation variables */ OSMNodes=NewNodeList(0,0); OSMWays=NewWayList(0,0); OSMRelations=NewRelationList(0,0); /* Create the error log file */ open_errorlog(FileName(dirname,prefix,"fixme.log"),0,option_keep); /* Parse the file */ for(arg=1;arg<argc;arg++) { int fd; char *filename,*p; if(argv[arg][0]=='-' && argv[arg][1]=='-') continue; filename=strcpy(malloc(strlen(argv[arg])+1),argv[arg]); fd=OpenFile(filename); if((p=strstr(filename,".bz2")) && !strcmp(p,".bz2")) { fd=Uncompress_Bzip2(fd); *p=0; } if((p=strstr(filename,".gz")) && !strcmp(p,".gz")) { fd=Uncompress_Gzip(fd); *p=0; } if((p=strstr(filename,".xz")) && !strcmp(p,".xz")) { fd=Uncompress_Xz(fd); *p=0; } printf("\nParse OSM Data [%s]\n==============\n\n",filename); fflush(stdout); if((p=strstr(filename,".pbf")) && !strcmp(p,".pbf")) { if(ParsePBFFile(fd,OSMNodes,OSMWays,OSMRelations)) exit(EXIT_FAILURE); } else if((p=strstr(filename,".o5m")) && !strcmp(p,".o5m")) { if(ParseO5MFile(fd,OSMNodes,OSMWays,OSMRelations)) exit(EXIT_FAILURE); } else { if(ParseOSMFile(fd,OSMNodes,OSMWays,OSMRelations)) exit(EXIT_FAILURE); } CloseFile(fd); free(filename); } DeleteXMLTaggingRules(); FinishNodeList(OSMNodes); FinishWayList(OSMWays); FinishRelationList(OSMRelations); /* Sort the data */ printf("\nSort OSM Data\n=============\n\n"); fflush(stdout); /* Sort the nodes, ways and relations */ SortNodeList(OSMNodes); SortWayList(OSMWays); SortRelationList(OSMRelations); /* Process the data */ RenameFile(OSMNodes->filename_tmp,OSMNodes->filename); RenameFile(OSMWays->filename_tmp,OSMWays->filename); RenameFile(OSMRelations->rrfilename_tmp,OSMRelations->rrfilename); RenameFile(OSMRelations->trfilename_tmp,OSMRelations->trfilename); close_errorlog(); printf("\nCreate Error Log\n================\n\n"); fflush(stdout); OSMErrorLogs=NewErrorLogList(); ProcessErrorLogs(OSMErrorLogs,OSMNodes,OSMWays,OSMRelations); SortErrorLogsGeographically(OSMErrorLogs); SaveErrorLogs(OSMErrorLogs,FileName(dirname,prefix,"fixme.mem")); FreeErrorLogList(OSMErrorLogs); /* Free the memory (delete the temporary files) */ FreeNodeList(OSMNodes,0); FreeWayList(OSMWays,0); FreeRelationList(OSMRelations,0); printf("\n"); fflush(stdout); printf_program_end(); exit(EXIT_SUCCESS); }
void PruneIsolatedRegions(NodesX *nodesx,SegmentsX *segmentsx,WaysX *waysx,distance_t minimum) { WaysX *newwaysx; WayX tmpwayx; transport_t transport; BitMask *connected,*region; index_t *regionsegments,*othersegments; index_t nallocregionsegments,nallocothersegments; if(nodesx->number==0 || segmentsx->number==0) return; /* Map into memory / open the files */ #if !SLIM nodesx->data=MapFile(nodesx->filename_tmp); segmentsx->data=MapFileWriteable(segmentsx->filename_tmp); waysx->data=MapFile(waysx->filename_tmp); #else nodesx->fd=SlimMapFile(nodesx->filename_tmp); segmentsx->fd=SlimMapFileWriteable(segmentsx->filename_tmp); waysx->fd=SlimMapFile(waysx->filename_tmp); InvalidateNodeXCache(nodesx->cache); InvalidateSegmentXCache(segmentsx->cache); InvalidateWayXCache(waysx->cache); #endif newwaysx=NewWayList(0,0); CloseFileBuffered(newwaysx->fd); newwaysx->fd=SlimMapFileWriteable(newwaysx->filename_tmp); connected=AllocBitMask(segmentsx->number); region =AllocBitMask(segmentsx->number); logassert(connected,"Failed to allocate memory (try using slim mode?)"); /* Check AllocBitMask() worked */ logassert(region,"Failed to allocate memory (try using slim mode?)"); /* Check AllocBitMask() worked */ regionsegments=(index_t*)malloc((nallocregionsegments=1024)*sizeof(index_t)); othersegments =(index_t*)malloc((nallocothersegments =1024)*sizeof(index_t)); logassert(regionsegments,"Failed to allocate memory (try using slim mode?)"); /* Check malloc() worked */ logassert(othersegments,"Failed to allocate memory (try using slim mode?)"); /* Check malloc() worked */ /* Loop through the transport types */ for(transport=Transport_None+1;transport<Transport_Count;transport++) { index_t i,j; index_t nregions=0,npruned=0,nadjusted=0; const char *transport_str=TransportName(transport); transports_t transports=TRANSPORTS(transport); if(!(waysx->allow&transports)) continue; /* Print the start message */ printf_first("Pruning Isolated Regions (%s): Segments=0 Adjusted=0 Pruned=0",transport_str); /* Loop through the segments and find the disconnected ones */ ClearAllBits(connected,segmentsx->number); ClearAllBits(region ,segmentsx->number); for(i=0;i<segmentsx->number;i++) { index_t nregionsegments=0,nothersegments=0; distance_t total=0; SegmentX *segmentx; WayX *wayx; if(IsBitSet(connected,i)) goto endloop; segmentx=LookupSegmentX(segmentsx,i,1); if(IsPrunedSegmentX(segmentx)) goto endloop; if(segmentx->way<waysx->number) wayx=LookupWayX(waysx,segmentx->way,1); else SlimFetch(newwaysx->fd,(wayx=&tmpwayx),sizeof(WayX),(segmentx->way-waysx->number)*sizeof(WayX)); if(!(wayx->way.allow&transports)) goto endloop; othersegments[nothersegments++]=i; SetBit(region,i); do { index_t thissegment,nodes[2]; thissegment=othersegments[--nothersegments]; if(nregionsegments==nallocregionsegments) regionsegments=(index_t*)realloc(regionsegments,(nallocregionsegments+=1024)*sizeof(index_t)); regionsegments[nregionsegments++]=thissegment; segmentx=LookupSegmentX(segmentsx,thissegment,1); nodes[0]=segmentx->node1; nodes[1]=segmentx->node2; total+=DISTANCE(segmentx->distance); for(j=0;j<2;j++) { NodeX *nodex=LookupNodeX(nodesx,nodes[j],1); if(!(nodex->allow&transports)) continue; segmentx=FirstSegmentX(segmentsx,nodes[j],1); while(segmentx) { index_t segment=IndexSegmentX(segmentsx,segmentx); if(segment!=thissegment) { if(segmentx->way<waysx->number) wayx=LookupWayX(waysx,segmentx->way,1); else SlimFetch(newwaysx->fd,(wayx=&tmpwayx),sizeof(WayX),(segmentx->way-waysx->number)*sizeof(WayX)); if(wayx->way.allow&transports) { /* Already connected - finish */ if(IsBitSet(connected,segment)) { total=minimum; goto foundconnection; } /* Not in region - add to list */ if(!IsBitSet(region,segment)) { if(nothersegments==nallocothersegments) othersegments=(index_t*)realloc(othersegments,(nallocothersegments+=1024)*sizeof(index_t)); othersegments[nothersegments++]=segment; SetBit(region,segment); } } } segmentx=NextSegmentX(segmentsx,segmentx,nodes[j]); } } } while(nothersegments>0 && total<minimum); foundconnection: /* Prune the segments or mark them as connected */ if(total<minimum) /* not connected - delete them */ { nregions++; for(j=0;j<nregionsegments;j++) { SegmentX *segmentx; WayX *wayx,tmpwayx; SetBit(connected,regionsegments[j]); /* not really connected, but don't need to check again */ ClearBit(region,regionsegments[j]); segmentx=LookupSegmentX(segmentsx,regionsegments[j],1); if(segmentx->way<waysx->number) wayx=LookupWayX(waysx,segmentx->way,1); else SlimFetch(newwaysx->fd,(wayx=&tmpwayx),sizeof(WayX),(segmentx->way-waysx->number)*sizeof(WayX)); if(wayx->way.allow==transports) { prune_segment(segmentsx,segmentx); npruned++; } else { if(segmentx->way<waysx->number) /* create a new way */ { tmpwayx=*wayx; tmpwayx.way.allow&=~transports; segmentx->way=waysx->number+newwaysx->number; SlimReplace(newwaysx->fd,&tmpwayx,sizeof(WayX),(segmentx->way-waysx->number)*sizeof(WayX)); newwaysx->number++; PutBackSegmentX(segmentsx,segmentx); } else /* modify the existing one */ { tmpwayx.way.allow&=~transports; SlimReplace(newwaysx->fd,&tmpwayx,sizeof(WayX),(segmentx->way-waysx->number)*sizeof(WayX)); } nadjusted++; } } } else /* connected - mark as part of the main region */ { for(j=0;j<nregionsegments;j++) { SetBit(connected,regionsegments[j]); ClearBit(region,regionsegments[j]); } for(j=0;j<nothersegments;j++) { SetBit(connected,othersegments[j]); ClearBit(region,othersegments[j]); } } endloop: if(!((i+1)%10000)) printf_middle("Pruning Isolated Regions (%s): Segments=%"Pindex_t" Adjusted=%"Pindex_t" Pruned=%"Pindex_t" (%"Pindex_t" Regions)",transport_str,i+1,nadjusted,npruned,nregions); } /* Print the final message */ printf_last("Pruned Isolated Regions (%s): Segments=%"Pindex_t" Adjusted=%"Pindex_t" Pruned=%"Pindex_t" (%"Pindex_t" Regions)",transport_str,segmentsx->number,nadjusted,npruned,nregions); } /* Unmap from memory / close the files */ free(region); free(connected); free(regionsegments); free(othersegments); #if !SLIM nodesx->data=UnmapFile(nodesx->data); segmentsx->data=UnmapFile(segmentsx->data); waysx->data=UnmapFile(waysx->data); #else nodesx->fd=SlimUnmapFile(nodesx->fd); segmentsx->fd=SlimUnmapFile(segmentsx->fd); waysx->fd=SlimUnmapFile(waysx->fd); #endif SlimUnmapFile(newwaysx->fd); waysx->number+=newwaysx->number; waysx->fd=OpenFileBufferedAppend(waysx->filename_tmp); newwaysx->fd=ReOpenFileBuffered(newwaysx->filename_tmp); while(!ReadFileBuffered(newwaysx->fd,&tmpwayx,sizeof(WayX))) WriteFileBuffered(waysx->fd,&tmpwayx,sizeof(WayX)); CloseFileBuffered(waysx->fd); CloseFileBuffered(newwaysx->fd); FreeWayList(newwaysx,0); }
int main(int argc,char** argv) { NodesX *Nodes; SegmentsX *Segments,*SuperSegments=NULL,*MergedSegments=NULL; WaysX *Ways; int iteration=0,quit=0; int max_iterations=10; char *dirname=NULL,*prefix=NULL,*tagging=NULL; int option_parse_only=0,option_process_only=0; int option_filenames=0; int arg; /* Parse the command line arguments */ for(arg=1;arg<argc;arg++) { if(!strcmp(argv[arg],"--help")) print_usage(1); else if(!strcmp(argv[arg],"--slim")) option_slim=1; else if(!strncmp(argv[arg],"--sort-ram-size=",16)) option_filesort_ramsize=atoi(&argv[arg][16]); else if(!strncmp(argv[arg],"--dir=",6)) dirname=&argv[arg][6]; else if(!strncmp(argv[arg],"--tmpdir=",9)) option_tmpdirname=&argv[arg][9]; else if(!strncmp(argv[arg],"--prefix=",9)) prefix=&argv[arg][9]; else if(!strcmp(argv[arg],"--parse-only")) option_parse_only=1; else if(!strcmp(argv[arg],"--process-only")) option_process_only=1; else if(!strncmp(argv[arg],"--max-iterations=",17)) max_iterations=atoi(&argv[arg][17]); else if(!strncmp(argv[arg],"--tagging=",10)) tagging=&argv[arg][10]; else if(argv[arg][0]=='-' && argv[arg][1]=='-') print_usage(0); else option_filenames++; } /* Check the specified command line options */ if(option_parse_only && option_process_only) print_usage(0); if(option_filenames && option_process_only) print_usage(0); if(!option_filesort_ramsize) { if(option_slim) option_filesort_ramsize=64*1024*1024; else option_filesort_ramsize=256*1024*1024; } else option_filesort_ramsize*=1024*1024; if(!option_tmpdirname) { if(!dirname) option_tmpdirname="."; else option_tmpdirname=dirname; } if(tagging && ExistsFile(tagging)) ; else if(!tagging && ExistsFile(FileName(dirname,prefix,"tagging.xml"))) tagging=FileName(dirname,prefix,"tagging.xml"); if(tagging && ParseXMLTaggingRules(tagging)) { fprintf(stderr,"Error: Cannot read the tagging rules in the file '%s'.\n",tagging); return(1); } if(!tagging) { fprintf(stderr,"Error: Cannot run without reading some tagging rules.\n"); return(1); } /* Create new node, segment and way variables */ Nodes=NewNodeList(option_parse_only||option_process_only); Segments=NewSegmentList(option_parse_only||option_process_only); Ways=NewWayList(option_parse_only||option_process_only); /* Parse the file */ if(option_filenames) { for(arg=1;arg<argc;arg++) { FILE *file; if(argv[arg][0]=='-' && argv[arg][1]=='-') continue; file=fopen(argv[arg],"rb"); if(!file) { fprintf(stderr,"Cannot open file '%s' for reading [%s].\n",argv[arg],strerror(errno)); exit(EXIT_FAILURE); } printf("\nParse OSM Data [%s]\n==============\n\n",argv[arg]); fflush(stdout); if(ParseOSM(file,Nodes,Segments,Ways)) exit(EXIT_FAILURE); fclose(file); } } else if(!option_process_only) { printf("\nParse OSM Data\n==============\n\n"); fflush(stdout); if(ParseOSM(stdin,Nodes,Segments,Ways)) exit(EXIT_FAILURE); } if(option_parse_only) { FreeNodeList(Nodes,1); FreeSegmentList(Segments,1); FreeWayList(Ways,1); return(0); } /* Process the data */ printf("\nProcess OSM Data\n================\n\n"); fflush(stdout); /* Sort the nodes, segments and ways */ SortNodeList(Nodes); SortSegmentList(Segments); SortWayList(Ways); /* Remove bad segments (must be after sorting the nodes and segments) */ RemoveBadSegments(Nodes,Segments); /* Remove non-highway nodes (must be after removing the bad segments) */ RemoveNonHighwayNodes(Nodes,Segments); /* Measure the segments and replace node/way id with index (must be after removing non-highway nodes) */ UpdateSegments(Segments,Nodes,Ways); /* Repeated iteration on Super-Nodes and Super-Segments */ do { printf("\nProcess Super-Data (iteration %d)\n================================%s\n\n",iteration,iteration>9?"=":""); fflush(stdout); if(iteration==0) { /* Select the super-nodes */ ChooseSuperNodes(Nodes,Segments,Ways); /* Select the super-segments */ SuperSegments=CreateSuperSegments(Nodes,Segments,Ways,iteration); } else { SegmentsX *SuperSegments2; /* Select the super-nodes */ ChooseSuperNodes(Nodes,SuperSegments,Ways); /* Select the super-segments */ SuperSegments2=CreateSuperSegments(Nodes,SuperSegments,Ways,iteration); if(SuperSegments->xnumber==SuperSegments2->xnumber) quit=1; FreeSegmentList(SuperSegments,0); SuperSegments=SuperSegments2; } /* Sort the super-segments */ SortSegmentList(SuperSegments); /* Remove duplicated super-segments */ DeduplicateSegments(SuperSegments,Nodes,Ways); iteration++; if(iteration>max_iterations) quit=1; } while(!quit); /* Combine the super-segments */ printf("\nCombine Segments and Super-Segments\n===================================\n\n"); fflush(stdout); /* Merge the super-segments */ MergedSegments=MergeSuperSegments(Segments,SuperSegments); FreeSegmentList(Segments,0); FreeSegmentList(SuperSegments,0); Segments=MergedSegments; /* Rotate segments so that node1<node2 */ RotateSegments(Segments); /* Sort the segments */ SortSegmentList(Segments); /* Remove duplicated segments */ DeduplicateSegments(Segments,Nodes,Ways); /* Cross reference the nodes and segments */ printf("\nCross-Reference Nodes and Segments\n==================================\n\n"); fflush(stdout); /* Sort the node list geographically */ SortNodeListGeographically(Nodes); /* Create the real segments and nodes */ CreateRealNodes(Nodes,iteration); CreateRealSegments(Segments,Ways); /* Fix the segment and node indexes */ IndexNodes(Nodes,Segments); IndexSegments(Segments,Nodes); /* Output the results */ printf("\nWrite Out Database Files\n========================\n\n"); fflush(stdout); /* Write out the nodes */ SaveNodeList(Nodes,FileName(dirname,prefix,"nodes.mem")); FreeNodeList(Nodes,0); /* Write out the segments */ SaveSegmentList(Segments,FileName(dirname,prefix,"segments.mem")); FreeSegmentList(Segments,0); /* Write out the ways */ SaveWayList(Ways,FileName(dirname,prefix,"ways.mem")); FreeWayList(Ways,0); return(0); }