static void *filesort_fixed_heapsort_thread(thread_data *thread)
{
int fd;
size_t item;
/* Sort the data pointers using a heap sort */
filesort_heapsort(thread->datap,thread->n,thread->compare);
/* Create a temporary file and write the result */
fd=OpenFileBufferedNew(thread->filename);
for(item=0;item<thread->n;item++)
WriteFileBuffered(fd,thread->datap[item],thread->itemsize);
CloseFileBuffered(fd);
#if defined(USE_PTHREADS) && USE_PTHREADS
if(option_filesort_threads>1)
{
pthread_mutex_lock(&running_mutex);
thread->running=2;
pthread_cond_signal(&running_cond);
pthread_mutex_unlock(&running_mutex);
}
#endif
return(NULL);
}
WaysX *NewWayList(int append,int readonly)
{
WaysX *waysx;
waysx=(WaysX*)calloc(1,sizeof(WaysX));
logassert(waysx,"Failed to allocate memory (try using slim mode?)"); /* Check calloc() worked */
waysx->filename =(char*)malloc(strlen(option_tmpdirname)+32);
waysx->filename_tmp=(char*)malloc(strlen(option_tmpdirname)+32);
sprintf(waysx->filename ,"%s/waysx.parsed.mem",option_tmpdirname);
sprintf(waysx->filename_tmp,"%s/waysx.%p.tmp" ,option_tmpdirname,(void*)waysx);
if(append || readonly)
if(ExistsFile(waysx->filename))
{
FILESORT_VARINT waysize;
int fd;
fd=ReOpenFileBuffered(waysx->filename);
while(!ReadFileBuffered(fd,&waysize,FILESORT_VARSIZE))
{
SkipFileBuffered(fd,waysize);
waysx->number++;
}
CloseFileBuffered(fd);
RenameFile(waysx->filename,waysx->filename_tmp);
}
if(append)
waysx->fd=OpenFileBufferedAppend(waysx->filename_tmp);
else if(!readonly)
waysx->fd=OpenFileBufferedNew(waysx->filename_tmp);
else
waysx->fd=-1;
#if SLIM
waysx->cache=NewWayXCache();
#endif
waysx->nfilename_tmp=(char*)malloc(strlen(option_tmpdirname)+32);
sprintf(waysx->nfilename_tmp,"%s/waynames.%p.tmp",option_tmpdirname,(void*)waysx);
return(waysx);
}
void CompactWayList(WaysX *waysx,SegmentsX *segmentsx)
{
int fd;
index_t cnumber;
if(waysx->number==0)
return;
/* Print the start message */
printf_first("Sorting Ways and Compacting");
/* Allocate the array of indexes */
waysx->cdata=(index_t*)malloc(waysx->number*sizeof(index_t));
logassert(waysx->cdata,"Failed to allocate memory (try using slim mode?)"); /* Check malloc() worked */
/* Re-open the file read-only and a new file writeable */
waysx->fd=ReOpenFileBuffered(waysx->filename_tmp);
DeleteFile(waysx->filename_tmp);
fd=OpenFileBufferedNew(waysx->filename_tmp);
/* Sort the ways to allow compacting according to the properties */
sortwaysx=waysx;
sortsegmentsx=segmentsx;
cnumber=filesort_fixed(waysx->fd,fd,sizeof(WayX),(int (*)(void*,index_t))delete_unused,
(int (*)(const void*,const void*))sort_by_name_and_prop_and_id,
(int (*)(void*,index_t))deduplicate_and_index_by_compact_id);
/* Close the files */
waysx->fd=CloseFileBuffered(waysx->fd);
CloseFileBuffered(fd);
/* Free the data */
free(segmentsx->usedway);
segmentsx->usedway=NULL;
/* Print the final message */
printf_last("Sorted and Compacted Ways: Ways=%"Pindex_t" Unique=%"Pindex_t,waysx->number,cnumber);
waysx->number=cnumber;
}
void SortWayList(WaysX *waysx)
{
index_t xnumber;
int fd;
/* Print the start message */
printf_first("Sorting Ways");
/* Re-open the file read-only and a new file writeable */
waysx->fd=ReOpenFileBuffered(waysx->filename_tmp);
DeleteFile(waysx->filename_tmp);
fd=OpenFileBufferedNew(waysx->filename_tmp);
/* Allocate the array of indexes */
waysx->idata=(way_t*)malloc(waysx->number*sizeof(way_t));
logassert(waysx->idata,"Failed to allocate memory (try using slim mode?)"); /* Check malloc() worked */
/* Sort the ways by ID and index them */
sortwaysx=waysx;
xnumber=waysx->number;
waysx->number=filesort_vary(waysx->fd,fd,NULL,
(int (*)(const void*,const void*))sort_by_id,
(int (*)(void*,index_t))deduplicate_and_index_by_id);
waysx->knumber=waysx->number;
/* Close the files */
waysx->fd=CloseFileBuffered(waysx->fd);
CloseFileBuffered(fd);
/* Print the final message */
printf_last("Sorted Ways: Ways=%"Pindex_t" Duplicates=%"Pindex_t,xnumber,xnumber-waysx->number);
}
void open_errorlog(const char *filename,int append,int bin)
{
/* Text log file */
errorlogfilename=(char*)malloc(strlen(filename)+8);
strcpy(errorlogfilename,filename);
#if defined(_MSC_VER) || defined(__MINGW32__)
errorlogfile=fopen(errorlogfilename,append?"ab":"wb");
#else
errorlogfile=fopen(errorlogfilename,append?"a" :"w" );
#endif
if(!errorlogfile)
{
fprintf(stderr,"Cannot open file '%s' for writing [%s].\n",errorlogfilename,strerror(errno));
exit(EXIT_FAILURE);
}
/* Binary log file */
if(bin)
{
errorbinfilename=(char*)malloc(strlen(filename)+8);
sprintf(errorbinfilename,"%s.tmp",filename);
errorfileoffset=0;
if(append)
{
if(ExistsFile(filename))
errorfileoffset=SizeFile(filename);
errorbinfile=OpenFileBufferedAppend(errorbinfilename);
}
else
errorbinfile=OpenFileBufferedNew(errorbinfilename);
}
else
errorbinfile=-1;
}
void SaveWayList(WaysX *waysx,const char *filename)
{
index_t i;
int fd;
index_t position=0;
WayX wayx;
WaysFile waysfile={0};
highways_t highways=0;
transports_t allow=0;
properties_t props=0;
/* Print the start message */
printf_first("Writing Ways: Ways=0");
/* Re-open the files */
waysx->fd=ReOpenFileBuffered(waysx->filename_tmp);
waysx->nfd=ReOpenFileBuffered(waysx->nfilename_tmp);
/* Write out the ways data */
fd=OpenFileBufferedNew(filename);
SeekFileBuffered(fd,sizeof(WaysFile));
for(i=0;i<waysx->number;i++)
{
ReadFileBuffered(waysx->fd,&wayx,sizeof(WayX));
highways|=HIGHWAYS(wayx.way.type);
allow |=wayx.way.allow;
props |=wayx.way.props;
WriteFileBuffered(fd,&wayx.way,sizeof(Way));
if(!((i+1)%1000))
printf_middle("Writing Ways: Ways=%"Pindex_t,i+1);
}
/* Write out the ways names */
SeekFileBuffered(fd,sizeof(WaysFile)+(off_t)waysx->number*sizeof(Way));
while(position<waysx->nlength)
{
size_t len=1024;
char temp[1024];
if((waysx->nlength-position)<1024)
len=waysx->nlength-position;
ReadFileBuffered(waysx->nfd,temp,len);
WriteFileBuffered(fd,temp,len);
position+=len;
}
/* Close the files */
waysx->fd=CloseFileBuffered(waysx->fd);
waysx->nfd=CloseFileBuffered(waysx->nfd);
/* Write out the header structure */
waysfile.number =waysx->number;
waysfile.highways=highways;
waysfile.allow =allow;
waysfile.props =props;
SeekFileBuffered(fd,0);
WriteFileBuffered(fd,&waysfile,sizeof(WaysFile));
CloseFileBuffered(fd);
/* Print the final message */
printf_last("Wrote Ways: Ways=%"Pindex_t,waysx->number);
}
void SortWayNames(WaysX *waysx)
{
index_t i;
int nfd;
char *names[2]={NULL,NULL};
int namelen[2]={0,0};
int nnames=0;
uint32_t lastlength=0;
/* Print the start message */
printf_first("Sorting Way Names");
/* Re-open the file read-only and new file writeable */
waysx->nfd=ReOpenFileBuffered(waysx->nfilename_tmp);
DeleteFile(waysx->nfilename_tmp);
nfd=OpenFileBufferedNew(waysx->nfilename_tmp);
/* Sort the way names */
waysx->nlength=0;
filesort_vary(waysx->nfd,nfd,NULL,
(int (*)(const void*,const void*))sort_by_name,
NULL);
/* Close the files */
waysx->nfd=CloseFileBuffered(waysx->nfd);
CloseFileBuffered(nfd);
/* Print the final message */
printf_last("Sorted Way Names: Ways=%"Pindex_t,waysx->number);
/* Print the start message */
printf_first("Updating Ways with Names: Ways=0 Names=0");
/* Map into memory / open the file */
#if !SLIM
waysx->data=MapFileWriteable(waysx->filename_tmp);
#else
waysx->fd=SlimMapFileWriteable(waysx->filename_tmp);
#endif
/* Re-open the file read-only and new file writeable */
waysx->nfd=ReOpenFileBuffered(waysx->nfilename_tmp);
DeleteFile(waysx->nfilename_tmp);
nfd=OpenFileBufferedNew(waysx->nfilename_tmp);
/* Update the ways and de-duplicate the names */
for(i=0;i<waysx->number;i++)
{
WayX *wayx;
index_t index;
FILESORT_VARINT size;
ReadFileBuffered(waysx->nfd,&size,FILESORT_VARSIZE);
if(namelen[nnames%2]<size)
names[nnames%2]=(char*)realloc((void*)names[nnames%2],namelen[nnames%2]=size);
ReadFileBuffered(waysx->nfd,&index,sizeof(index_t));
ReadFileBuffered(waysx->nfd,names[nnames%2],size-sizeof(index_t));
if(nnames==0 || strcmp(names[0],names[1]))
{
WriteFileBuffered(nfd,names[nnames%2],size-sizeof(index_t));
lastlength=waysx->nlength;
waysx->nlength+=size-sizeof(index_t);
nnames++;
}
wayx=LookupWayX(waysx,index,1);
wayx->way.name=lastlength;
PutBackWayX(waysx,wayx);
if(!((i+1)%1000))
printf_middle("Updating Ways with Names: Ways=%"Pindex_t" Names=%"Pindex_t,i+1,nnames);
}
if(names[0]) free(names[0]);
if(names[1]) free(names[1]);
/* Close the files */
waysx->nfd=CloseFileBuffered(waysx->nfd);
CloseFileBuffered(nfd);
/* Unmap from memory / close the files */
#if !SLIM
waysx->data=UnmapFile(waysx->data);
#else
waysx->fd=SlimUnmapFile(waysx->fd);
#endif
/* Print the final message */
printf_last("Updated Ways with Names: Ways=%"Pindex_t" Names=%"Pindex_t,waysx->number,nnames);
}
SegmentsX *SplitWays(WaysX *waysx,NodesX *nodesx,int keep)
{
SegmentsX *segmentsx;
index_t i;
int fd,nfd;
char *name=NULL;
int namelen=0;
/* Print the start message */
printf_first("Splitting Ways: Ways=0 Segments=0");
segmentsx=NewSegmentList();
/* Re-open the file read-only and a new file writeable */
waysx->fd=ReOpenFileBuffered(waysx->filename_tmp);
if(keep)
RenameFile(waysx->filename_tmp,waysx->filename);
else
DeleteFile(waysx->filename_tmp);
fd=OpenFileBufferedNew(waysx->filename_tmp);
nfd=OpenFileBufferedNew(waysx->nfilename_tmp);
/* Loop through the ways and create the segments and way names */
for(i=0;i<waysx->number;i++)
{
WayX wayx;
FILESORT_VARINT size;
node_t node,prevnode=NO_NODE_ID;
index_t index,previndex=NO_NODE;
ReadFileBuffered(waysx->fd,&size,FILESORT_VARSIZE);
ReadFileBuffered(waysx->fd,&wayx,sizeof(WayX));
waysx->allow|=wayx.way.allow;
while(!ReadFileBuffered(waysx->fd,&node,sizeof(node_t)) && node!=NO_NODE_ID)
{
index=IndexNodeX(nodesx,node);
if(prevnode==node)
{
logerror("Way %"Pway_t" contains node %"Pnode_t" that is connected to itself.\n",logerror_way(wayx.id),logerror_node(node));
}
else if(index==NO_NODE)
{
logerror("Way %"Pway_t" contains node %"Pnode_t" that does not exist in the Routino database.\n",logerror_way(wayx.id),logerror_node(node));
}
else if(previndex==NO_NODE)
;
else
{
distance_t segment_flags=0;
if(wayx.way.type&Highway_OneWay)
segment_flags|=ONEWAY_1TO2;
if(wayx.way.type&Highway_Area)
segment_flags|=SEGMENT_AREA;
AppendSegmentList(segmentsx,i,previndex,index,segment_flags);
}
prevnode=node;
previndex=index;
size-=sizeof(node_t);
}
size-=sizeof(node_t)+sizeof(WayX);
if(namelen<size)
name=(char*)realloc((void*)name,namelen=size);
ReadFileBuffered(waysx->fd,name,size);
WriteFileBuffered(fd,&wayx,sizeof(WayX));
size+=sizeof(index_t);
WriteFileBuffered(nfd,&size,FILESORT_VARSIZE);
WriteFileBuffered(nfd,&i,sizeof(index_t));
WriteFileBuffered(nfd,name,size-sizeof(index_t));
if(!((i+1)%1000))
printf_middle("Splitting Ways: Ways=%"Pindex_t" Segments=%"Pindex_t,i+1,segmentsx->number);
}
FinishSegmentList(segmentsx);
if(name) free(name);
/* Close the files */
waysx->fd=CloseFileBuffered(waysx->fd);
CloseFileBuffered(fd);
CloseFileBuffered(nfd);
/* Print the final message */
printf_last("Split Ways: Ways=%"Pindex_t" Segments=%"Pindex_t,waysx->number,segmentsx->number);
return(segmentsx);
}
