static void *filesort_fixed_heapsort_thread(thread_data *thread) { int fd; size_t item; /* Sort the data pointers using a heap sort */ filesort_heapsort(thread->datap,thread->n,thread->compare); /* Create a temporary file and write the result */ fd=OpenFileBufferedNew(thread->filename); for(item=0;item<thread->n;item++) WriteFileBuffered(fd,thread->datap[item],thread->itemsize); CloseFileBuffered(fd); #if defined(USE_PTHREADS) && USE_PTHREADS if(option_filesort_threads>1) { pthread_mutex_lock(&running_mutex); thread->running=2; pthread_cond_signal(&running_cond); pthread_mutex_unlock(&running_mutex); } #endif return(NULL); }
WaysX *NewWayList(int append,int readonly) { WaysX *waysx; waysx=(WaysX*)calloc(1,sizeof(WaysX)); logassert(waysx,"Failed to allocate memory (try using slim mode?)"); /* Check calloc() worked */ waysx->filename =(char*)malloc(strlen(option_tmpdirname)+32); waysx->filename_tmp=(char*)malloc(strlen(option_tmpdirname)+32); sprintf(waysx->filename ,"%s/waysx.parsed.mem",option_tmpdirname); sprintf(waysx->filename_tmp,"%s/waysx.%p.tmp" ,option_tmpdirname,(void*)waysx); if(append || readonly) if(ExistsFile(waysx->filename)) { FILESORT_VARINT waysize; int fd; fd=ReOpenFileBuffered(waysx->filename); while(!ReadFileBuffered(fd,&waysize,FILESORT_VARSIZE)) { SkipFileBuffered(fd,waysize); waysx->number++; } CloseFileBuffered(fd); RenameFile(waysx->filename,waysx->filename_tmp); } if(append) waysx->fd=OpenFileBufferedAppend(waysx->filename_tmp); else if(!readonly) waysx->fd=OpenFileBufferedNew(waysx->filename_tmp); else waysx->fd=-1; #if SLIM waysx->cache=NewWayXCache(); #endif waysx->nfilename_tmp=(char*)malloc(strlen(option_tmpdirname)+32); sprintf(waysx->nfilename_tmp,"%s/waynames.%p.tmp",option_tmpdirname,(void*)waysx); return(waysx); }
void CompactWayList(WaysX *waysx,SegmentsX *segmentsx) { int fd; index_t cnumber; if(waysx->number==0) return; /* Print the start message */ printf_first("Sorting Ways and Compacting"); /* Allocate the array of indexes */ waysx->cdata=(index_t*)malloc(waysx->number*sizeof(index_t)); logassert(waysx->cdata,"Failed to allocate memory (try using slim mode?)"); /* Check malloc() worked */ /* Re-open the file read-only and a new file writeable */ waysx->fd=ReOpenFileBuffered(waysx->filename_tmp); DeleteFile(waysx->filename_tmp); fd=OpenFileBufferedNew(waysx->filename_tmp); /* Sort the ways to allow compacting according to the properties */ sortwaysx=waysx; sortsegmentsx=segmentsx; cnumber=filesort_fixed(waysx->fd,fd,sizeof(WayX),(int (*)(void*,index_t))delete_unused, (int (*)(const void*,const void*))sort_by_name_and_prop_and_id, (int (*)(void*,index_t))deduplicate_and_index_by_compact_id); /* Close the files */ waysx->fd=CloseFileBuffered(waysx->fd); CloseFileBuffered(fd); /* Free the data */ free(segmentsx->usedway); segmentsx->usedway=NULL; /* Print the final message */ printf_last("Sorted and Compacted Ways: Ways=%"Pindex_t" Unique=%"Pindex_t,waysx->number,cnumber); waysx->number=cnumber; }
void SortWayList(WaysX *waysx) { index_t xnumber; int fd; /* Print the start message */ printf_first("Sorting Ways"); /* Re-open the file read-only and a new file writeable */ waysx->fd=ReOpenFileBuffered(waysx->filename_tmp); DeleteFile(waysx->filename_tmp); fd=OpenFileBufferedNew(waysx->filename_tmp); /* Allocate the array of indexes */ waysx->idata=(way_t*)malloc(waysx->number*sizeof(way_t)); logassert(waysx->idata,"Failed to allocate memory (try using slim mode?)"); /* Check malloc() worked */ /* Sort the ways by ID and index them */ sortwaysx=waysx; xnumber=waysx->number; waysx->number=filesort_vary(waysx->fd,fd,NULL, (int (*)(const void*,const void*))sort_by_id, (int (*)(void*,index_t))deduplicate_and_index_by_id); waysx->knumber=waysx->number; /* Close the files */ waysx->fd=CloseFileBuffered(waysx->fd); CloseFileBuffered(fd); /* Print the final message */ printf_last("Sorted Ways: Ways=%"Pindex_t" Duplicates=%"Pindex_t,xnumber,xnumber-waysx->number); }
void open_errorlog(const char *filename,int append,int bin) { /* Text log file */ errorlogfilename=(char*)malloc(strlen(filename)+8); strcpy(errorlogfilename,filename); #if defined(_MSC_VER) || defined(__MINGW32__) errorlogfile=fopen(errorlogfilename,append?"ab":"wb"); #else errorlogfile=fopen(errorlogfilename,append?"a" :"w" ); #endif if(!errorlogfile) { fprintf(stderr,"Cannot open file '%s' for writing [%s].\n",errorlogfilename,strerror(errno)); exit(EXIT_FAILURE); } /* Binary log file */ if(bin) { errorbinfilename=(char*)malloc(strlen(filename)+8); sprintf(errorbinfilename,"%s.tmp",filename); errorfileoffset=0; if(append) { if(ExistsFile(filename)) errorfileoffset=SizeFile(filename); errorbinfile=OpenFileBufferedAppend(errorbinfilename); } else errorbinfile=OpenFileBufferedNew(errorbinfilename); } else errorbinfile=-1; }
void SaveWayList(WaysX *waysx,const char *filename) { index_t i; int fd; index_t position=0; WayX wayx; WaysFile waysfile={0}; highways_t highways=0; transports_t allow=0; properties_t props=0; /* Print the start message */ printf_first("Writing Ways: Ways=0"); /* Re-open the files */ waysx->fd=ReOpenFileBuffered(waysx->filename_tmp); waysx->nfd=ReOpenFileBuffered(waysx->nfilename_tmp); /* Write out the ways data */ fd=OpenFileBufferedNew(filename); SeekFileBuffered(fd,sizeof(WaysFile)); for(i=0;i<waysx->number;i++) { ReadFileBuffered(waysx->fd,&wayx,sizeof(WayX)); highways|=HIGHWAYS(wayx.way.type); allow |=wayx.way.allow; props |=wayx.way.props; WriteFileBuffered(fd,&wayx.way,sizeof(Way)); if(!((i+1)%1000)) printf_middle("Writing Ways: Ways=%"Pindex_t,i+1); } /* Write out the ways names */ SeekFileBuffered(fd,sizeof(WaysFile)+(off_t)waysx->number*sizeof(Way)); while(position<waysx->nlength) { size_t len=1024; char temp[1024]; if((waysx->nlength-position)<1024) len=waysx->nlength-position; ReadFileBuffered(waysx->nfd,temp,len); WriteFileBuffered(fd,temp,len); position+=len; } /* Close the files */ waysx->fd=CloseFileBuffered(waysx->fd); waysx->nfd=CloseFileBuffered(waysx->nfd); /* Write out the header structure */ waysfile.number =waysx->number; waysfile.highways=highways; waysfile.allow =allow; waysfile.props =props; SeekFileBuffered(fd,0); WriteFileBuffered(fd,&waysfile,sizeof(WaysFile)); CloseFileBuffered(fd); /* Print the final message */ printf_last("Wrote Ways: Ways=%"Pindex_t,waysx->number); }
void SortWayNames(WaysX *waysx) { index_t i; int nfd; char *names[2]={NULL,NULL}; int namelen[2]={0,0}; int nnames=0; uint32_t lastlength=0; /* Print the start message */ printf_first("Sorting Way Names"); /* Re-open the file read-only and new file writeable */ waysx->nfd=ReOpenFileBuffered(waysx->nfilename_tmp); DeleteFile(waysx->nfilename_tmp); nfd=OpenFileBufferedNew(waysx->nfilename_tmp); /* Sort the way names */ waysx->nlength=0; filesort_vary(waysx->nfd,nfd,NULL, (int (*)(const void*,const void*))sort_by_name, NULL); /* Close the files */ waysx->nfd=CloseFileBuffered(waysx->nfd); CloseFileBuffered(nfd); /* Print the final message */ printf_last("Sorted Way Names: Ways=%"Pindex_t,waysx->number); /* Print the start message */ printf_first("Updating Ways with Names: Ways=0 Names=0"); /* Map into memory / open the file */ #if !SLIM waysx->data=MapFileWriteable(waysx->filename_tmp); #else waysx->fd=SlimMapFileWriteable(waysx->filename_tmp); #endif /* Re-open the file read-only and new file writeable */ waysx->nfd=ReOpenFileBuffered(waysx->nfilename_tmp); DeleteFile(waysx->nfilename_tmp); nfd=OpenFileBufferedNew(waysx->nfilename_tmp); /* Update the ways and de-duplicate the names */ for(i=0;i<waysx->number;i++) { WayX *wayx; index_t index; FILESORT_VARINT size; ReadFileBuffered(waysx->nfd,&size,FILESORT_VARSIZE); if(namelen[nnames%2]<size) names[nnames%2]=(char*)realloc((void*)names[nnames%2],namelen[nnames%2]=size); ReadFileBuffered(waysx->nfd,&index,sizeof(index_t)); ReadFileBuffered(waysx->nfd,names[nnames%2],size-sizeof(index_t)); if(nnames==0 || strcmp(names[0],names[1])) { WriteFileBuffered(nfd,names[nnames%2],size-sizeof(index_t)); lastlength=waysx->nlength; waysx->nlength+=size-sizeof(index_t); nnames++; } wayx=LookupWayX(waysx,index,1); wayx->way.name=lastlength; PutBackWayX(waysx,wayx); if(!((i+1)%1000)) printf_middle("Updating Ways with Names: Ways=%"Pindex_t" Names=%"Pindex_t,i+1,nnames); } if(names[0]) free(names[0]); if(names[1]) free(names[1]); /* Close the files */ waysx->nfd=CloseFileBuffered(waysx->nfd); CloseFileBuffered(nfd); /* Unmap from memory / close the files */ #if !SLIM waysx->data=UnmapFile(waysx->data); #else waysx->fd=SlimUnmapFile(waysx->fd); #endif /* Print the final message */ printf_last("Updated Ways with Names: Ways=%"Pindex_t" Names=%"Pindex_t,waysx->number,nnames); }
SegmentsX *SplitWays(WaysX *waysx,NodesX *nodesx,int keep) { SegmentsX *segmentsx; index_t i; int fd,nfd; char *name=NULL; int namelen=0; /* Print the start message */ printf_first("Splitting Ways: Ways=0 Segments=0"); segmentsx=NewSegmentList(); /* Re-open the file read-only and a new file writeable */ waysx->fd=ReOpenFileBuffered(waysx->filename_tmp); if(keep) RenameFile(waysx->filename_tmp,waysx->filename); else DeleteFile(waysx->filename_tmp); fd=OpenFileBufferedNew(waysx->filename_tmp); nfd=OpenFileBufferedNew(waysx->nfilename_tmp); /* Loop through the ways and create the segments and way names */ for(i=0;i<waysx->number;i++) { WayX wayx; FILESORT_VARINT size; node_t node,prevnode=NO_NODE_ID; index_t index,previndex=NO_NODE; ReadFileBuffered(waysx->fd,&size,FILESORT_VARSIZE); ReadFileBuffered(waysx->fd,&wayx,sizeof(WayX)); waysx->allow|=wayx.way.allow; while(!ReadFileBuffered(waysx->fd,&node,sizeof(node_t)) && node!=NO_NODE_ID) { index=IndexNodeX(nodesx,node); if(prevnode==node) { logerror("Way %"Pway_t" contains node %"Pnode_t" that is connected to itself.\n",logerror_way(wayx.id),logerror_node(node)); } else if(index==NO_NODE) { logerror("Way %"Pway_t" contains node %"Pnode_t" that does not exist in the Routino database.\n",logerror_way(wayx.id),logerror_node(node)); } else if(previndex==NO_NODE) ; else { distance_t segment_flags=0; if(wayx.way.type&Highway_OneWay) segment_flags|=ONEWAY_1TO2; if(wayx.way.type&Highway_Area) segment_flags|=SEGMENT_AREA; AppendSegmentList(segmentsx,i,previndex,index,segment_flags); } prevnode=node; previndex=index; size-=sizeof(node_t); } size-=sizeof(node_t)+sizeof(WayX); if(namelen<size) name=(char*)realloc((void*)name,namelen=size); ReadFileBuffered(waysx->fd,name,size); WriteFileBuffered(fd,&wayx,sizeof(WayX)); size+=sizeof(index_t); WriteFileBuffered(nfd,&size,FILESORT_VARSIZE); WriteFileBuffered(nfd,&i,sizeof(index_t)); WriteFileBuffered(nfd,name,size-sizeof(index_t)); if(!((i+1)%1000)) printf_middle("Splitting Ways: Ways=%"Pindex_t" Segments=%"Pindex_t,i+1,segmentsx->number); } FinishSegmentList(segmentsx); if(name) free(name); /* Close the files */ waysx->fd=CloseFileBuffered(waysx->fd); CloseFileBuffered(fd); CloseFileBuffered(nfd); /* Print the final message */ printf_last("Split Ways: Ways=%"Pindex_t" Segments=%"Pindex_t,waysx->number,segmentsx->number); return(segmentsx); }