/* tries to re-open the bz stream at the next stream start.
returns 0 on success, -1 on failure. */
int bzReOpen(struct Input *ctx, int *error) {
/* for copying out the last unused part of the block which
has an EOS token in it. needed for re-initialising the
next stream. */
unsigned char unused[BZ_MAX_UNUSED];
void *unused_tmp_ptr = NULL;
int nUnused, i;
BZ2_bzReadGetUnused(error, (BZFILE *)(ctx->fileHandle), &unused_tmp_ptr, &nUnused);
if (*error != BZ_OK) return -1;
/* when bzReadClose is called the unused buffer is deallocated,
so it needs to be copied somewhere safe first. */
for (i = 0; i < nUnused; ++i)
unused[i] = ((unsigned char *)unused_tmp_ptr)[i];
BZ2_bzReadClose(error, (BZFILE *)(ctx->fileHandle));
if (*error != BZ_OK) return -1;
/* reassign the file handle */
ctx->fileHandle = BZ2_bzReadOpen(error, ctx->systemHandle, 0, 0, unused, nUnused);
if (ctx->fileHandle == NULL || *error != BZ_OK) return -1;
return 0;
}
void BZ2Stream::read(void* ptr, size_t size) {
BZ2_bzRead(&bzerror_, bzfile_, ptr, size);
advanceOffset(size);
switch (bzerror_) {
case BZ_OK: return;
case BZ_STREAM_END:
if (getUnused() || getUnusedLength() > 0)
ROS_ERROR("unused data already available");
else {
char* unused;
int nUnused;
BZ2_bzReadGetUnused(&bzerror_, bzfile_, (void**) &unused, &nUnused);
setUnused(unused);
setUnusedLength(nUnused);
}
return;
case BZ_IO_ERROR: throw BagIOException("BZ_IO_ERROR: error reading from compressed stream"); break;
case BZ_UNEXPECTED_EOF: throw BagIOException("BZ_UNEXPECTED_EOF: compressed stream ended before logical end-of-stream detected"); break;
case BZ_DATA_ERROR: throw BagIOException("BZ_DATA_ERROR: data integrity error detected in compressed stream"); break;
case BZ_DATA_ERROR_MAGIC: throw BagIOException("BZ_DATA_ERROR_MAGIC: stream does not begin with requisite header bytes"); break;
case BZ_MEM_ERROR: throw BagIOException("BZ_MEM_ERROR: insufficient memory available"); break;
}
}
size_t AFILE_afread(void *ptr, size_t size, size_t count, AFILE *afp)
{
#ifndef DISABLE_BZLIB
int32_t nbuf=0, i;
void *unused_tmp_void=NULL;
char *unused_tmp=NULL;
#endif
switch(afp->c) {
case AFILE_NO_COMPRESSION:
return fread(ptr, size, count, afp->fp);
#ifndef DISABLE_BZLIB
case AFILE_BZ2_COMPRESSION:
while(0 == nbuf &&
!(BZ_STREAM_END == afp->bzerror && 0 == afp->n_unused && feof(afp->fp))) {
nbuf = BZ2_bzRead(&afp->bzerror, afp->bz2, ptr, size*count);
if(BZ_OK == afp->bzerror) {
// return # of bytes
return nbuf;
}
else if(BZ_STREAM_END == afp->bzerror) {
// Get unused
BZ2_bzReadGetUnused(&afp->bzerror, afp->bz2, &unused_tmp_void, &afp->n_unused);
if(BZ_OK != afp->bzerror) AFILE_print_error("Could not BZ2_bzReadGetUnused");
unused_tmp = (char*)unused_tmp_void;
for(i=0;i<afp->n_unused;i++) {
afp->unused[i] = unused_tmp[i];
}
// Close
BZ2_bzReadClose(&afp->bzerror, afp->bz2);
if(BZ_OK != afp->bzerror) AFILE_print_error("Could not BZ2_bzReadClose");
afp->bzerror = BZ_STREAM_END; // set to the stream end for next call to this function
// Open again if necessary
if(0 == afp->n_unused && feof(afp->fp)) {
return nbuf;
}
else {
afp->bz2 = BZ2_bzReadOpen(&afp->bzerror, afp->fp, 0, 0, afp->unused, afp->n_unused);
if(NULL == afp->bz2) AFILE_print_error("Could not open file");
}
}
else {
fprintf(stderr, "nbuf = %d\n", nbuf);
fprintf(stderr, "afp->bzerror = %d\n", afp->bzerror);
AFILE_print_error("Could not read");
}
}
return nbuf;
#endif
case AFILE_GZ_COMPRESSION:
return gzread(afp->gz, ptr, size*count);
break;
default:
AFILE_print_error("Could not recognize compresssion\n");
break;
}
return 0;
}
size_t
bread (void *ptr, size_t size, size_t nitems, void *file)
{
struct fhandle *f = (struct fhandle *) file;
size_t ret;
ret = BZ2_bzRead (&bzerror, (BZFILE *) f->file2, ptr, size * nitems);
if (bzerror == BZ_STREAM_END)
{
char unused[1024];
int nunused = sizeof (unused);
BZ2_bzReadGetUnused (&bzerror, (BZFILE *) f->file2,
(void **)&unused, &nunused);
}
return ret;
}
int fsReadCompressed(FILESYSTEM_FILE *f, void *data, int len) {
BZFILE *bzf;
int err, back;
void *buf;
if (!f)
return -1;
if (!(bzf = BZ2_bzReadOpen(&err, f->fp, 0, 0, NULL, 0)))
return -1;
BZ2_bzRead(&err, bzf, data, len);
if (err != BZ_STREAM_END) {
BZ2_bzReadClose(&err, bzf);
return -1;
}
BZ2_bzReadGetUnused(&err, bzf, &buf, &back);
fseek(f->fp, back * -1, SEEK_CUR);
f->pos = ftell(f->fp) - f->offset;
BZ2_bzReadClose(&err, bzf);
return 0;
}
/* Thread to read from the compressed file and perform bzip2 uncompression */
static void *start_file_read_thread(void *data)
{
struct osm_planet *osf = data;
unsigned char curr = 0;
while(!osf->exit_now)
{
int bzerror;
/* If there is no buffer available for writing, wait until the main
* thread signals that it has just drained one of the buffers. */
pthread_mutex_lock(&osf->drained_mutex);
if(osf->buff_filled[curr] && !osf->exit_now)
pthread_cond_wait(&osf->drained_signal, &osf->drained_mutex);
pthread_mutex_unlock(&osf->drained_mutex);
if(osf->exit_now)
break;
/* Decompress up to 900000 bytes into the current buffer and store the
* number of bytes actually decoded, which may be less in the unlikely
* event that the bzip2 compression hasn't been done with the maximum
* block size (which is 900000 bytes). */
osf->buff_len[curr] = BZ2_bzRead(&bzerror, osf->bzfp, osf->buff[curr], BLOCK_SIZE);
/* Mark the current buffer as filled and signal to the main thread that
* it is now available for reading. */
osf->buff_filled[curr] = 1;
curr = !curr;
pthread_mutex_lock(&osf->filled_mutex);
pthread_cond_signal(&osf->filled_signal);
pthread_mutex_unlock(&osf->filled_mutex);
if(bzerror == BZ_STREAM_END) /* end of compression block */
{
void *unused;
int num_unused;
/* If this is also end of file, then exit now... */
if(feof(osf->fp))
{
fprintf(stderr, "End of file\n");
break;
}
/* ...otherwise try reopening the stream to read the next block. */
BZ2_bzReadGetUnused(&bzerror, osf->bzfp, &unused, &num_unused);
fseek(osf->fp, -num_unused, SEEK_CUR);
BZ2_bzReadClose(&bzerror, osf->bzfp);
osf->bzfp = BZ2_bzReadOpen(&bzerror, osf->fp, 1, 0, NULL, 0);
}
if(bzerror != BZ_OK)
{
fprintf(stderr, "Error reading from compressed OSM file: %d\n", bzerror);
break;
}
}
fprintf(stderr, "File read thread exiting...\n");
osf->finished = 1;
return NULL;
}
char*BzReader::readLine(char*s, int n){
int error=BZ_OK;
int verbosity=0;
int small=0;
if(m_bzFile==NULL && !feof(m_file)){
#ifdef __bz2_verbose__
cout<<"Opening bz2 file"<<endl;
#endif
m_bzFile=BZ2_bzReadOpen(&error,m_file,verbosity,small,
m_unused,m_nUnused);
if(error!=BZ_OK)
cout<<"Error: BZ2_bzReadOpen failed."<<endl;
}
//cout<<"[BzReader::readLine]"<<endl;
#ifdef CONFIG_ASSERT
if(!(n<=__BzReader_MAXIMUM_LENGTH)){
cout<<"Expected: "<<__BzReader_MAXIMUM_LENGTH<<" Actual: "<<n<<endl;
}
assert(n<=__BzReader_MAXIMUM_LENGTH);
#endif
int pos=-1;
for(int i=m_bufferPosition;i<m_bufferSize;i++){
if(m_buffer[i]=='\n'){
pos=i;
}
}
if(pos!=-1){
int i=0;
while(m_buffer[m_bufferPosition]!='\n' && m_bufferPosition<m_bufferSize){
if(i==n){
s[i]='\0';
return s;
}
s[i++]=m_buffer[m_bufferPosition++];
}
if(i==n){
s[i]='\0';
return s;
}
if(m_bufferPosition<m_bufferSize){
s[i++]=m_buffer[m_bufferPosition++];
}
s[i]='\0';
return s;
}
/* copy the leftover of buffer in an other buffer */
int i=0;
while(m_bufferPosition<m_bufferSize){
if(i==n){
s[i]='\0';
return s;
}
s[i++]=m_buffer[m_bufferPosition++];
}
/* read some bytes from the compressed file */
m_bufferPosition=0;
m_bufferSize=0;
if(m_bzFile!=NULL)
m_bufferSize=BZ2_bzRead(&error,m_bzFile,m_buffer,__BzReader_MAXIMUM_LENGTH);
if(error==BZ_STREAM_END){
#ifdef __bz2_verbose__
// no more thing to read.
cout<<"Notice: BZ2_bzRead returned BZ_STREAM_END"<<endl;
cout<<"Total bytes: "<<m_bytesLoaded<<endl;
#endif
// get unused bytes for the next round
BZ2_bzReadGetUnused ( &error, m_bzFile, &m_unused1, &m_nUnused );
if(error!=BZ_OK)
cout<<"Error with BZ2_bzReadGetUnused"<<endl;
// copy unused bytes
memcpy(m_unused,m_unused1,m_nUnused);
#ifdef __bz2_verbose__
cout<<"Closing file for now."<<endl;
#endif
BZ2_bzReadClose ( &error, m_bzFile );
m_bzFile=NULL;
#ifdef __bz2_verbose__
if(feof(m_file))
cout<<"Reached end of file, bufferSize is "<<m_bufferSize<<endl;
#endif
// nothing more is available.
if(m_bufferSize==0 && feof(m_file)){
return NULL;
}
}else if(error!=BZ_OK){
cout<<"Error: BZ2_bzRead did not return BZ_OK or BZ_STREAM_END."<<endl;
cout<<"bzFile= "<<m_bzFile<<endl;
processError(error);
return NULL;
}
m_bytesLoaded+=m_bufferSize;
/* copy up to \n (including it) into secondaryBuffer */
while(i<n && m_buffer[m_bufferPosition]!='\n' && m_bufferPosition<m_bufferSize){
if(i==n){
s[i]='\0';
return s;
}
s[i++]=m_buffer[m_bufferPosition++];
}
// don't return more than n bytes
if(i==n){
s[i]='\0';
return s;
}
if( i < n && m_bufferPosition<m_bufferSize){
s[i++]=m_buffer[m_bufferPosition++];
}
s[i]='\0';
// Nothing loaded.
if(i==0)
return NULL;
return s;
}
