void PNGAPI png_read_init_3(png_structpp ptr_ptr, png_const_charp user_png_ver, png_size_t png_struct_size) { #ifdef PNG_SETJMP_SUPPORTED jmp_buf tmp_jmp; /* to save current jump buffer */ #endif int i=0; png_structp png_ptr=*ptr_ptr; do { if(user_png_ver[i] != png_libpng_ver[i]) { #ifdef PNG_LEGACY_SUPPORTED png_ptr->flags |= PNG_FLAG_LIBRARY_MISMATCH; #else png_ptr->warning_fn=(png_error_ptr)NULL; png_warning(png_ptr, "Application uses deprecated png_read_init() and should be recompiled."); break; #endif } } while (png_libpng_ver[i++]); png_debug(1, "in png_read_init_3\n"); #ifdef PNG_SETJMP_SUPPORTED /* save jump buffer and error functions */ png_memcpy(tmp_jmp, png_ptr->jmpbuf, sizeof (jmp_buf)); #endif if(sizeof(png_struct) > png_struct_size) { png_destroy_struct(png_ptr); *ptr_ptr = (png_structp)png_create_struct(PNG_STRUCT_PNG); png_ptr = *ptr_ptr; } /* reset all variables to 0 */ png_memset(png_ptr, 0, sizeof (png_struct)); #ifdef PNG_SETJMP_SUPPORTED /* restore jump buffer */ png_memcpy(png_ptr->jmpbuf, tmp_jmp, sizeof (jmp_buf)); #endif /* initialize zbuf - compression buffer */ png_ptr->zbuf_size = PNG_ZBUF_SIZE; png_ptr->zbuf = (png_bytep)png_malloc(png_ptr, (png_uint_32)png_ptr->zbuf_size); png_ptr->zstream.zalloc = png_zalloc; png_ptr->zstream.zfree = png_zfree; png_ptr->zstream.opaque = (voidpf)png_ptr; switch (inflateInit(&png_ptr->zstream)) { case Z_OK: /* Do nothing */ break; case Z_MEM_ERROR: case Z_STREAM_ERROR: png_error(png_ptr, "zlib memory"); break; case Z_VERSION_ERROR: png_error(png_ptr, "zlib version"); break; default: png_error(png_ptr, "Unknown zlib error"); } png_ptr->zstream.next_out = png_ptr->zbuf; png_ptr->zstream.avail_out = (uInt)png_ptr->zbuf_size; png_set_read_fn(png_ptr, NULL, NULL); }
void extractBLKX(AbstractFile* in, AbstractFile* out, BLKXTable* blkx) { unsigned char* inBuffer; unsigned char* outBuffer; unsigned char zero; size_t bufferSize; size_t have; off_t initialOffset; int i; int ret; z_stream strm; bufferSize = SECTOR_SIZE * blkx->decompressBufferRequested; ASSERT(inBuffer = (unsigned char*) malloc(bufferSize), "malloc"); ASSERT(outBuffer = (unsigned char*) malloc(bufferSize), "malloc"); initialOffset = out->tell(out); ASSERT(initialOffset != -1, "ftello"); zero = 0; for(i = 0; i < blkx->blocksRunCount; i++) { ASSERT(in->seek(in, blkx->dataStart + blkx->runs[i].compOffset) == 0, "fseeko"); ASSERT(out->seek(out, initialOffset + (blkx->runs[i].sectorStart * SECTOR_SIZE)) == 0, "mSeek"); if(blkx->runs[i].sectorCount > 0) { ASSERT(out->seek(out, initialOffset + (blkx->runs[i].sectorStart + blkx->runs[i].sectorCount) * SECTOR_SIZE - 1) == 0, "mSeek"); ASSERT(out->write(out, &zero, 1) == 1, "mWrite"); ASSERT(out->seek(out, initialOffset + (blkx->runs[i].sectorStart * SECTOR_SIZE)) == 0, "mSeek"); } if(blkx->runs[i].type == BLOCK_TERMINATOR) { break; } if( blkx->runs[i].compLength == 0) { continue; } printf("run %d: start=%" PRId64 " sectors=%" PRId64 ", length=%" PRId64 ", fileOffset=0x%" PRIx64 "\n", i, initialOffset + (blkx->runs[i].sectorStart * SECTOR_SIZE), blkx->runs[i].sectorCount, blkx->runs[i].compLength, blkx->runs[i].compOffset); switch(blkx->runs[i].type) { case BLOCK_ZLIB: strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; strm.avail_in = 0; strm.next_in = Z_NULL; ASSERT(inflateInit(&strm) == Z_OK, "inflateInit"); ASSERT((strm.avail_in = in->read(in, inBuffer, blkx->runs[i].compLength)) == blkx->runs[i].compLength, "fread"); strm.next_in = inBuffer; do { strm.avail_out = bufferSize; strm.next_out = outBuffer; ASSERT((ret = inflate(&strm, Z_NO_FLUSH)) != Z_STREAM_ERROR, "inflate/Z_STREAM_ERROR"); if(ret != Z_OK && ret != Z_BUF_ERROR && ret != Z_STREAM_END) { ASSERT(FALSE, "inflate"); } have = bufferSize - strm.avail_out; ASSERT(out->write(out, outBuffer, have) == have, "mWrite"); } while (strm.avail_out == 0); ASSERT(inflateEnd(&strm) == Z_OK, "inflateEnd"); break; case BLOCK_RAW: if(blkx->runs[i].compLength > bufferSize) { uint64_t left = blkx->runs[i].compLength; void* pageBuffer = malloc(DEFAULT_BUFFER_SIZE); while(left > 0) { size_t thisRead; if(left > DEFAULT_BUFFER_SIZE) { thisRead = DEFAULT_BUFFER_SIZE; } else { thisRead = left; } ASSERT((have = in->read(in, pageBuffer, thisRead)) == thisRead, "fread"); ASSERT(out->write(out, pageBuffer, have) == have, "mWrite"); left -= have; } free(pageBuffer); } else { ASSERT((have = in->read(in, inBuffer, blkx->runs[i].compLength)) == blkx->runs[i].compLength, "fread"); ASSERT(out->write(out, inBuffer, have) == have, "mWrite"); } break; case BLOCK_IGNORE: break; case BLOCK_COMMENT: break; case BLOCK_TERMINATOR: break; default: break; } } free(inBuffer); free(outBuffer); }
indri::api::ParsedDocument* indri::collection::CompressedCollection::retrieve( int documentID ) { indri::thread::ScopedLock l( _lock ); UINT64 offset; int actual; if( !_lookup.get( documentID, &offset, actual, sizeof offset ) ) { LEMUR_THROW( LEMUR_IO_ERROR, "Unable to find document " + i64_to_string(documentID) + " in the collection." ); } // flush output buffer; make sure all data is on disk if( _output ) _output->flush(); // decompress the data indri::utility::Buffer output; z_stream_s stream; stream.zalloc = zlib_alloc; stream.zfree = zlib_free; inflateInit( &stream ); zlib_read_document( stream, _storage, offset, output ); int decompressedSize = stream.total_out; // initialize the buffer as a ParsedDocument indri::api::ParsedDocument* document = (indri::api::ParsedDocument*) output.front(); new(document) indri::api::ParsedDocument; document->text = 0; document->textLength = 0; document->content = 0; document->contentLength = 0; // get the number of fields (it's the last byte) char* dataStart = output.front() + sizeof(indri::api::ParsedDocument); int fieldCount = copy_quad( dataStart + decompressedSize - 4 ); int endOffset = decompressedSize - 4 - 2*fieldCount*sizeof(UINT32); char* arrayStart = dataStart + endOffset; const char* positionData = 0; int positionDataLength = 0; // store metadata for( int i=0; i<fieldCount; i++ ) { int keyStart = copy_quad( arrayStart + 2*i*sizeof(UINT32) ); int valueStart = copy_quad( arrayStart + (2*i+1)*sizeof(UINT32) ); int valueEnd; if( i==(fieldCount-1) ) { valueEnd = endOffset; } else { valueEnd = copy_quad( arrayStart + 2*(i+1)*sizeof(UINT32) ); } indri::parse::MetadataPair pair; pair.key = dataStart + keyStart; pair.value = dataStart + valueStart; pair.valueLength = valueEnd - valueStart; // extract text if( !strcmp( pair.key, TEXT_KEY ) ) { document->text = (char*) pair.value; document->textLength = pair.valueLength; } // extract content if( !strcmp( pair.key, CONTENT_KEY ) ) { document->content = document->text + copy_quad( (char*) pair.value ); } // extract content length if( !strcmp( pair.key, CONTENTLENGTH_KEY ) ) { document->contentLength = copy_quad( (char *)pair.value ); } if( !strcmp( pair.key, POSITIONS_KEY ) ) { positionData = (char*) pair.value; positionDataLength = pair.valueLength; } document->metadata.push_back( pair ); } // decompress positions _readPositions( document, positionData, positionDataLength ); output.detach(); return document; }
static int zlib_inflate(FILE *fin, FILE *fout) { z_stream strm; unsigned char bin[ZLIB_CHUNK_SIZE]; unsigned char bout[ZLIB_CHUNK_SIZE]; int nleft, result; strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; strm.avail_in = 0; strm.next_in = Z_NULL; result = inflateInit(&strm); if (result != Z_OK) return result; do { strm.avail_in = fread(bin, 1, ZLIB_CHUNK_SIZE, fin); if (ferror(fin)) { inflateEnd(&strm); return Z_ERRNO; } if (strm.avail_in == 0) break; strm.next_in = bin; do { strm.avail_out = ZLIB_CHUNK_SIZE; strm.next_out = bout; result = inflate(&strm, Z_NO_FLUSH); if (result == Z_STREAM_ERROR) return result; switch (result) { case Z_NEED_DICT: result = Z_DATA_ERROR; case Z_DATA_ERROR: case Z_MEM_ERROR: inflateEnd(&strm); return result; break; } nleft = ZLIB_CHUNK_SIZE - strm.avail_out; if ((fwrite(bout, 1, nleft, fout) != nleft) || ferror(fout)) { inflateEnd(&strm); return Z_ERRNO; } } while (strm.avail_out == 0); } while (result != Z_STREAM_END); inflateEnd(&strm); if (result != Z_STREAM_END) return Z_DATA_ERROR; return Z_OK; }
/** * \brief Loads a file from a pak/zip file into memory. * \param[in] hFile Pointer to a valid filehandle_t structure. * \param[in] pakfiles Pointer to a valid packfile_t structure. * \return true on success, otherwise false. */ PRIVATE _boolean LoadCompressedFile( filehandle_t *hFile, packfile_t *pakfiles ) { int err; W32 read; W8 *uncompr; W8 *buf; z_stream d_stream; /* decompression stream */ buf = Z_Malloc( pakfiles->filelength + 2 ); // Zlib expects the 2 byte head that the inflate method adds. buf[ 0 ] = 120; buf[ 1 ] = 156; read = fread( buf+2, 1, pakfiles->filelength, hFile->hFile ); if( read != pakfiles->filelength ) { fclose( hFile->hFile ); Z_Free( buf ); return false; } fclose( hFile->hFile ); hFile->hFile = NULL; uncompr = Z_Malloc( pakfiles->uncompressed_length ); d_stream.zalloc = (alloc_func)0; d_stream.zfree = (free_func)0; d_stream.opaque = (voidpf)0; d_stream.next_in = buf; d_stream.avail_in = (uInt)pakfiles->filelength+2; d_stream.next_out = uncompr; d_stream.avail_out = (uInt)pakfiles->uncompressed_length; err = inflateInit( &d_stream ); if( err != Z_OK ) { MM_FREE( uncompr ); Z_Free( buf ); FS_CloseFile( hFile ); return false; } err = inflate( &d_stream, Z_NO_FLUSH ); if( err != Z_OK ) { Z_Free( uncompr ); Z_Free( buf ); FS_CloseFile( hFile ); return false; } err = inflateEnd( &d_stream ); if( err != Z_OK ) { Z_Free( uncompr ); Z_Free( buf ); FS_CloseFile( hFile ); return false; } Z_Free( buf ); hFile->filedata = uncompr; hFile->filesize = d_stream.total_out; // align our file data pointers hFile->ptrStart = hFile->ptrCurrent = (PW8)hFile->filedata; hFile->ptrEnd = (PW8)hFile->filedata + hFile->filesize; hFile->bLoaded = true; return true; }
/* Decompress from file source to file dest until stream ends or EOF. inf() returns Z_OK on success, Z_MEM_ERROR if memory could not be allocated for processing, Z_DATA_ERROR if the deflate data is invalid or incomplete, Z_VERSION_ERROR if the version of zlib.h and the version of the library linked do not match, or Z_ERRNO if there is an error reading or writing the files. */ bool Misc::inflate(const uint8_t* in, size_t inlen, std::ostream& dest, string& err, int CHUNK) { int ret; unsigned have; z_stream strm; if ( CHUNK == -1 ) CHUNK = CL_Z_DEFAULT_CHUNK; uint8_t* out = (uint8_t*)malloc(CHUNK); /* allocate inflate state */ strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; strm.avail_in = 0; strm.next_in = Z_NULL; ret = inflateInit(&strm); if (ret != Z_OK){ free(out); zerr(ret, err); return false; } /* decompress until deflate stream ends or end of file */ do { strm.avail_in = inlen; if (strm.avail_in == 0) break; strm.next_in = (uint8_t*)in; /* run inflate() on input until output buffer not full */ do { strm.avail_out = CHUNK; strm.next_out = out; ret = ::inflate(&strm, Z_NO_FLUSH); assert(ret != Z_STREAM_ERROR); /* state not clobbered */ switch (ret) { case Z_NEED_DICT: ret = Z_DATA_ERROR; /* and fall through */ case Z_DATA_ERROR: case Z_MEM_ERROR: (void)inflateEnd(&strm); free(out); zerr(ret, err); return false; } have = CHUNK - strm.avail_out; dest.write( (char*)out,have); if ( dest.fail() ) { (void)inflateEnd(&strm); free(out); zerr(Z_ERRNO, err); return false; } } while (strm.avail_out == 0); /* done when inflate() says it's done */ } while (ret != Z_STREAM_END); /* clean up and return */ (void)inflateEnd(&strm); free(out); if ( ret == Z_STREAM_END ) return true; zerr(Z_DATA_ERROR, err); return false; }
int FileInfo::Inf ( LFA_FileRef source, LFA_FileRef dest ) { int ret; unsigned have; z_stream strm; unsigned char in[CHUNK]; unsigned char out[CHUNK]; XMP_Uns64 allBytes = 0; // allocate inflate state strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; strm.avail_in = 0; strm.next_in = Z_NULL; ret = inflateInit ( &strm ); if ( ret != Z_OK ) return ret; // decompress until deflate stream ends or end of file LFA_Seek ( source, SWF_COMPRESSION_BEGIN, SEEK_SET ); XMP_Uns64 outPos = SWF_COMPRESSION_BEGIN; try { do { strm.avail_in = LFA_Read ( source, in, CHUNK ); if ( strm.avail_in == 0 ) { ret = Z_STREAM_END; break; } strm.next_in = in; // run inflate() on input until output buffer not full do { strm.avail_out = CHUNK; strm.next_out = out; ret = inflate ( &strm, Z_NO_FLUSH ); XMP_Assert ( ret != Z_STREAM_ERROR ); // state not clobbered switch ( ret ) { case Z_NEED_DICT: ret = Z_DATA_ERROR; // and fall through case Z_DATA_ERROR: case Z_MEM_ERROR: inflateEnd ( &strm ); return ret; } have = CHUNK - strm.avail_out; LFA_Seek ( dest, outPos, SEEK_SET ); LFA_Write ( dest, out, have ); outPos += have; } while ( strm.avail_out == 0 ); // done when inflate() says it's done } while ( ret != Z_STREAM_END ); } catch ( ... ) { inflateEnd ( &strm ); return Z_ERRNO; } // clean up and return inflateEnd ( &strm ); return ( (ret == Z_STREAM_END) ? Z_OK : Z_DATA_ERROR ); } // FileInfo::Inf
static int dmg_open(BlockDriverState *bs, QDict *options, int flags, Error **errp) { BDRVDMGState *s = bs->opaque; DmgHeaderState ds; uint64_t rsrc_fork_offset, rsrc_fork_length; uint64_t plist_xml_offset, plist_xml_length; int64_t offset; int ret; bs->read_only = 1; s->n_chunks = 0; s->offsets = s->lengths = s->sectors = s->sectorcounts = NULL; /* used by dmg_read_mish_block to keep track of the current I/O position */ ds.data_fork_offset = 0; ds.max_compressed_size = 1; ds.max_sectors_per_chunk = 1; /* locate the UDIF trailer */ offset = dmg_find_koly_offset(bs->file->bs, errp); if (offset < 0) { ret = offset; goto fail; } /* offset of data fork (DataForkOffset) */ ret = read_uint64(bs, offset + 0x18, &ds.data_fork_offset); if (ret < 0) { goto fail; } else if (ds.data_fork_offset > offset) { ret = -EINVAL; goto fail; } /* offset of resource fork (RsrcForkOffset) */ ret = read_uint64(bs, offset + 0x28, &rsrc_fork_offset); if (ret < 0) { goto fail; } ret = read_uint64(bs, offset + 0x30, &rsrc_fork_length); if (ret < 0) { goto fail; } if (rsrc_fork_offset >= offset || rsrc_fork_length > offset - rsrc_fork_offset) { ret = -EINVAL; goto fail; } /* offset of property list (XMLOffset) */ ret = read_uint64(bs, offset + 0xd8, &plist_xml_offset); if (ret < 0) { goto fail; } ret = read_uint64(bs, offset + 0xe0, &plist_xml_length); if (ret < 0) { goto fail; } if (plist_xml_offset >= offset || plist_xml_length > offset - plist_xml_offset) { ret = -EINVAL; goto fail; } ret = read_uint64(bs, offset + 0x1ec, (uint64_t *)&bs->total_sectors); if (ret < 0) { goto fail; } if (bs->total_sectors < 0) { ret = -EINVAL; goto fail; } if (rsrc_fork_length != 0) { ret = dmg_read_resource_fork(bs, &ds, rsrc_fork_offset, rsrc_fork_length); if (ret < 0) { goto fail; } } else if (plist_xml_length != 0) { ret = dmg_read_plist_xml(bs, &ds, plist_xml_offset, plist_xml_length); if (ret < 0) { goto fail; } } else { ret = -EINVAL; goto fail; } /* initialize zlib engine */ s->compressed_chunk = qemu_try_blockalign(bs->file->bs, ds.max_compressed_size + 1); s->uncompressed_chunk = qemu_try_blockalign(bs->file->bs, 512 * ds.max_sectors_per_chunk); if (s->compressed_chunk == NULL || s->uncompressed_chunk == NULL) { ret = -ENOMEM; goto fail; } if (inflateInit(&s->zstream) != Z_OK) { ret = -EINVAL; goto fail; } s->current_chunk = s->n_chunks; qemu_co_mutex_init(&s->lock); return 0; fail: g_free(s->types); g_free(s->offsets); g_free(s->lengths); g_free(s->sectors); g_free(s->sectorcounts); qemu_vfree(s->compressed_chunk); qemu_vfree(s->uncompressed_chunk); return ret; }
int main(int argc, char* argv[]) { //Discard existing output: FILE* fileo = fopen("output.txt", "w"); if (fileo) fclose(fileo); fileo = fopen("output.txt", "a"); //Open the PDF source file: FILE* filei = fopen("input.pdf", "rb"); if (filei && fileo) { //Get the file length: int fseekres = fseek(filei,0, SEEK_END); //fseek==0 if ok long filelen = ftell(filei); fseekres = fseek(filei,0, SEEK_SET); //Read ethe ntire file into memory (!): char* buffer = new char [filelen]; bzero(buffer, filelen);//ZeroMemory(buffer, filelen); size_t actualread = fread(buffer, filelen, 1 ,filei); //must return 1 bool morestreams = true; //Now search the buffer repeated for streams of data: while (morestreams) { //Search for stream, endstream. We ought to first check the filter //of the object to make sure it if FlateDecode, but skip that for now! size_t streamstart = FindStringInBuffer (buffer, "stream", filelen); size_t streamend = FindStringInBuffer (buffer, "endstream", filelen); if (streamstart>0 && streamend>streamstart) { //Skip to beginning and end of the data stream: streamstart += 6; if (buffer[streamstart]==0x0d && buffer[streamstart+1]==0x0a) streamstart+=2; else if (buffer[streamstart]==0x0a) streamstart++; if (buffer[streamend-2]==0x0d && buffer[streamend-1]==0x0a) streamend-=2; else if (buffer[streamend-1]==0x0a) streamend--; //Assume output will fit into 10 times input buffer: size_t outsize = (streamend - streamstart)*10; char* output = new char [outsize]; bzero(output, outsize);//ZeroMemory(output, outsize); //Now use zlib to inflate: z_stream zstrm; bzero(&zstrm, sizeof(zstrm));//ZeroMemory(&zstrm, sizeof(zstrm)); zstrm.avail_in = streamend - streamstart + 1; zstrm.avail_out = outsize; zstrm.next_in = (Bytef*)(buffer + streamstart); zstrm.next_out = (Bytef*)output; int rsti = inflateInit(&zstrm); if (rsti == Z_OK) { int rst2 = inflate (&zstrm, Z_FINISH); if (rst2 >= 0) { //Ok, got something, extract the text: size_t totout = zstrm.total_out; ProcessOutput(fileo, output, totout); } } delete[] output; output=0; buffer+= streamend + 7; filelen = filelen - (streamend+7); } else { morestreams = false; } } fclose(filei); } if (fileo) fclose(fileo); return 0; }
int cli_scanxar(cli_ctx *ctx) { int rc = CL_SUCCESS; unsigned int cksum_fails = 0; unsigned int extract_errors = 0; #if HAVE_LIBXML2 int fd = -1; struct xar_header hdr; fmap_t *map = *ctx->fmap; long length, offset, size, at; int encoding; z_stream strm; char *toc, *tmpname; xmlTextReaderPtr reader = NULL; int a_hash, e_hash; unsigned char *a_cksum = NULL, *e_cksum = NULL; void *a_hash_ctx = NULL, *e_hash_ctx = NULL; char result[SHA1_HASH_SIZE]; memset(&strm, 0x00, sizeof(z_stream)); /* retrieve xar header */ if (fmap_readn(*ctx->fmap, &hdr, 0, sizeof(hdr)) != sizeof(hdr)) { cli_dbgmsg("cli_scanxar: Invalid header, too short.\n"); return CL_EFORMAT; } hdr.magic = be32_to_host(hdr.magic); if (hdr.magic == XAR_HEADER_MAGIC) { cli_dbgmsg("cli_scanxar: Matched magic\n"); } else { cli_dbgmsg("cli_scanxar: Invalid magic\n"); return CL_EFORMAT; } hdr.size = be16_to_host(hdr.size); hdr.version = be16_to_host(hdr.version); hdr.toc_length_compressed = be64_to_host(hdr.toc_length_compressed); hdr.toc_length_decompressed = be64_to_host(hdr.toc_length_decompressed); hdr.chksum_alg = be32_to_host(hdr.chksum_alg); /* cli_dbgmsg("hdr.magic %x\n", hdr.magic); */ /* cli_dbgmsg("hdr.size %i\n", hdr.size); */ /* cli_dbgmsg("hdr.version %i\n", hdr.version); */ /* cli_dbgmsg("hdr.toc_length_compressed %lu\n", hdr.toc_length_compressed); */ /* cli_dbgmsg("hdr.toc_length_decompressed %lu\n", hdr.toc_length_decompressed); */ /* cli_dbgmsg("hdr.chksum_alg %i\n", hdr.chksum_alg); */ /* Uncompress TOC */ strm.next_in = (unsigned char *)fmap_need_off_once(*ctx->fmap, hdr.size, hdr.toc_length_compressed); if (strm.next_in == NULL) { cli_dbgmsg("cli_scanxar: fmap_need_off_once fails on TOC.\n"); return CL_EREAD; } strm.avail_in = hdr.toc_length_compressed; toc = cli_malloc(hdr.toc_length_decompressed+1); if (toc == NULL) { cli_dbgmsg("cli_scanxar: cli_malloc fails on TOC decompress buffer.\n"); return CL_EMEM; } toc[hdr.toc_length_decompressed] = '\0'; strm.avail_out = hdr.toc_length_decompressed; strm.next_out = (unsigned char *)toc; rc = inflateInit(&strm); if (rc != Z_OK) { cli_dbgmsg("cli_scanxar:inflateInit error %i \n", rc); rc = CL_EFORMAT; goto exit_toc; } rc = inflate(&strm, Z_SYNC_FLUSH); if (rc != Z_OK && rc != Z_STREAM_END) { cli_dbgmsg("cli_scanxar:inflate error %i \n", rc); rc = CL_EFORMAT; goto exit_toc; } rc = inflateEnd(&strm); if (rc != Z_OK) { cli_dbgmsg("cli_scanxar:inflateEnd error %i \n", rc); rc = CL_EFORMAT; goto exit_toc; } /* cli_dbgmsg("cli_scanxar: TOC xml:\n%s\n", toc); */ /* printf("cli_scanxar: TOC xml:\n%s\n", toc); */ /* cli_dbgmsg("cli_scanxar: TOC end:\n"); */ /* printf("cli_scanxar: TOC end:\n"); */ /* scan the xml */ cli_dbgmsg("cli_scanxar: scanning xar TOC xml in memory.\n"); rc = cli_mem_scandesc(toc, hdr.toc_length_decompressed, ctx); if (rc != CL_SUCCESS) { if (rc != CL_VIRUS || !SCAN_ALL) goto exit_toc; } /* make a file to leave if --leave-temps in effect */ if(ctx->engine->keeptmp) { if ((rc = cli_gentempfd(ctx->engine->tmpdir, &tmpname, &fd)) != CL_SUCCESS) { cli_dbgmsg("cli_scanxar: Can't create temporary file for TOC.\n"); goto exit_toc; } if (cli_writen(fd, toc, hdr.toc_length_decompressed) < 0) { cli_dbgmsg("cli_scanxar: cli_writen error writing TOC.\n"); rc = CL_EWRITE; xar_cleanup_temp_file(ctx, fd, tmpname); goto exit_toc; } rc = xar_cleanup_temp_file(ctx, fd, tmpname); if (rc != CL_SUCCESS) goto exit_toc; } reader = xmlReaderForMemory(toc, hdr.toc_length_decompressed, "noname.xml", NULL, CLAMAV_MIN_XMLREADER_FLAGS); if (reader == NULL) { cli_dbgmsg("cli_scanxar: xmlReaderForMemory error for TOC\n"); goto exit_toc; } rc = xar_scan_subdocuments(reader, ctx); if (rc != CL_SUCCESS) { cli_dbgmsg("xar_scan_subdocuments returns %i.\n", rc); goto exit_reader; } /* Walk the TOC XML and extract files */ fd = -1; tmpname = NULL; while (CL_SUCCESS == (rc = xar_get_toc_data_values(reader, &length, &offset, &size, &encoding, &a_cksum, &a_hash, &e_cksum, &e_hash))) { int do_extract_cksum = 1; unsigned char * blockp; void *a_sc, *e_sc; void *a_mc, *e_mc; char * expected; /* clean up temp file from previous loop iteration */ if (fd > -1 && tmpname) { rc = xar_cleanup_temp_file(ctx, fd, tmpname); if (rc != CL_SUCCESS) goto exit_reader; } at = offset + hdr.toc_length_compressed + hdr.size; if ((rc = cli_gentempfd(ctx->engine->tmpdir, &tmpname, &fd)) != CL_SUCCESS) { cli_dbgmsg("cli_scanxar: Can't generate temporary file.\n"); goto exit_reader; } cli_dbgmsg("cli_scanxar: decompress into temp file:\n%s, size %li,\n" "from xar heap offset %li length %li\n", tmpname, size, offset, length); a_hash_ctx = xar_hash_init(a_hash, &a_sc, &a_mc); e_hash_ctx = xar_hash_init(e_hash, &e_sc, &e_mc); switch (encoding) { case CL_TYPE_GZ: /* inflate gzip directly because file segments do not contain magic */ memset(&strm, 0, sizeof(strm)); if ((rc = inflateInit(&strm)) != Z_OK) { cli_dbgmsg("cli_scanxar: InflateInit failed: %d\n", rc); rc = CL_EFORMAT; extract_errors++; break; } while ((size_t)at < map->len && (unsigned long)at < offset+hdr.toc_length_compressed+hdr.size+length) { unsigned long avail_in; void * next_in; unsigned int bytes = MIN(map->len - at, map->pgsz); bytes = MIN(length, bytes); if(!(strm.next_in = next_in = (void*)fmap_need_off_once(map, at, bytes))) { cli_dbgmsg("cli_scanxar: Can't read %u bytes @ %lu.\n", bytes, (long unsigned)at); inflateEnd(&strm); rc = CL_EREAD; goto exit_tmpfile; } at += bytes; strm.avail_in = avail_in = bytes; do { int inf, outsize = 0; unsigned char buff[FILEBUFF]; strm.avail_out = sizeof(buff); strm.next_out = buff; inf = inflate(&strm, Z_SYNC_FLUSH); if (inf != Z_OK && inf != Z_STREAM_END && inf != Z_BUF_ERROR) { cli_dbgmsg("cli_scanxar: inflate error %i %s.\n", inf, strm.msg?strm.msg:""); rc = CL_EFORMAT; extract_errors++; break; } bytes = sizeof(buff) - strm.avail_out; if (e_hash_ctx != NULL) xar_hash_update(e_hash_ctx, buff, bytes, e_hash); if (cli_writen(fd, buff, bytes) < 0) { cli_dbgmsg("cli_scanxar: cli_writen error file %s.\n", tmpname); inflateEnd(&strm); rc = CL_EWRITE; goto exit_tmpfile; } outsize += sizeof(buff) - strm.avail_out; if (cli_checklimits("cli_scanxar", ctx, outsize, 0, 0) != CL_CLEAN) { break; } if (inf == Z_STREAM_END) { break; } } while (strm.avail_out == 0); if (rc != CL_SUCCESS) break; avail_in -= strm.avail_in; if (a_hash_ctx != NULL) xar_hash_update(a_hash_ctx, next_in, avail_in, a_hash); } inflateEnd(&strm); break; case CL_TYPE_7Z: #define CLI_LZMA_OBUF_SIZE 1024*1024 #define CLI_LZMA_HDR_SIZE LZMA_PROPS_SIZE+8 #define CLI_LZMA_IBUF_SIZE CLI_LZMA_OBUF_SIZE>>2 /* estimated compression ratio 25% */ { struct CLI_LZMA lz; unsigned long in_remaining = length; unsigned long out_size = 0; unsigned char * buff = __lzma_wrap_alloc(NULL, CLI_LZMA_OBUF_SIZE); int lret; memset(&lz, 0, sizeof(lz)); if (buff == NULL) { cli_dbgmsg("cli_scanxar: memory request for lzma decompression buffer fails.\n"); rc = CL_EMEM; goto exit_tmpfile; } blockp = (void*)fmap_need_off_once(map, at, CLI_LZMA_HDR_SIZE); if (blockp == NULL) { char errbuff[128]; cli_strerror(errno, errbuff, sizeof(errbuff)); cli_dbgmsg("cli_scanxar: Can't read %li bytes @ %li, errno:%s.\n", length, at, errbuff); rc = CL_EREAD; __lzma_wrap_free(NULL, buff); goto exit_tmpfile; } lz.next_in = blockp; lz.avail_in = CLI_LZMA_HDR_SIZE; if (a_hash_ctx != NULL) xar_hash_update(a_hash_ctx, blockp, CLI_LZMA_HDR_SIZE, a_hash); lret = cli_LzmaInit(&lz, 0); if (lret != LZMA_RESULT_OK) { cli_dbgmsg("cli_scanxar: cli_LzmaInit() fails: %i.\n", lret); rc = CL_EFORMAT; __lzma_wrap_free(NULL, buff); extract_errors++; break; } at += CLI_LZMA_HDR_SIZE; in_remaining -= CLI_LZMA_HDR_SIZE; while ((size_t)at < map->len && (unsigned long)at < offset+hdr.toc_length_compressed+hdr.size+length) { SizeT avail_in; SizeT avail_out; void * next_in; unsigned long in_consumed; lz.next_out = buff; lz.avail_out = CLI_LZMA_OBUF_SIZE; lz.avail_in = avail_in = MIN(CLI_LZMA_IBUF_SIZE, in_remaining); lz.next_in = next_in = (void*)fmap_need_off_once(map, at, lz.avail_in); if (lz.next_in == NULL) { char errbuff[128]; cli_strerror(errno, errbuff, sizeof(errbuff)); cli_dbgmsg("cli_scanxar: Can't read %li bytes @ %li, errno: %s.\n", length, at, errbuff); rc = CL_EREAD; __lzma_wrap_free(NULL, buff); cli_LzmaShutdown(&lz); goto exit_tmpfile; } lret = cli_LzmaDecode(&lz); if (lret != LZMA_RESULT_OK && lret != LZMA_STREAM_END) { cli_dbgmsg("cli_scanxar: cli_LzmaDecode() fails: %i.\n", lret); rc = CL_EFORMAT; extract_errors++; break; } in_consumed = avail_in - lz.avail_in; in_remaining -= in_consumed; at += in_consumed; avail_out = CLI_LZMA_OBUF_SIZE - lz.avail_out; if (avail_out == 0) cli_dbgmsg("cli_scanxar: cli_LzmaDecode() produces no output for " "avail_in %lu, avail_out %lu.\n", avail_in, avail_out); if (a_hash_ctx != NULL) xar_hash_update(a_hash_ctx, next_in, in_consumed, a_hash); if (e_hash_ctx != NULL) xar_hash_update(e_hash_ctx, buff, avail_out, e_hash); /* Write a decompressed block. */ /* cli_dbgmsg("Writing %li bytes to LZMA decompress temp file, " */ /* "consumed %li of %li available compressed bytes.\n", */ /* avail_out, in_consumed, avail_in); */ if (cli_writen(fd, buff, avail_out) < 0) { cli_dbgmsg("cli_scanxar: cli_writen error writing lzma temp file for %li bytes.\n", avail_out); __lzma_wrap_free(NULL, buff); cli_LzmaShutdown(&lz); rc = CL_EWRITE; goto exit_tmpfile; } /* Check file size limitation. */ out_size += avail_out; if (cli_checklimits("cli_scanxar", ctx, out_size, 0, 0) != CL_CLEAN) { break; } if (lret == LZMA_STREAM_END) break; } cli_LzmaShutdown(&lz); __lzma_wrap_free(NULL, buff); } break; case CL_TYPE_ANY: default: case CL_TYPE_BZ: case CL_TYPE_XZ: /* for uncompressed, bzip2, xz, and unknown, just pull the file, cli_magic_scandesc does the rest */ do_extract_cksum = 0; { unsigned long write_len; if (ctx->engine->maxfilesize) write_len = MIN((size_t)(ctx->engine->maxfilesize), (size_t)length); else write_len = length; if (!(blockp = (void*)fmap_need_off_once(map, at, length))) { char errbuff[128]; cli_strerror(errno, errbuff, sizeof(errbuff)); cli_dbgmsg("cli_scanxar: Can't read %li bytes @ %li, errno:%s.\n", length, at, errbuff); rc = CL_EREAD; goto exit_tmpfile; } if (a_hash_ctx != NULL) xar_hash_update(a_hash_ctx, blockp, length, a_hash); if (cli_writen(fd, blockp, write_len) < 0) { cli_dbgmsg("cli_scanxar: cli_writen error %li bytes @ %li.\n", length, at); rc = CL_EWRITE; goto exit_tmpfile; } /*break;*/ } } if (rc == CL_SUCCESS) { if (a_hash_ctx != NULL) { xar_hash_final(a_hash_ctx, result, a_hash); a_hash_ctx = NULL; } else { cli_dbgmsg("cli_scanxar: archived-checksum missing.\n"); cksum_fails++; } if (a_cksum != NULL) { expected = cli_hex2str((char *)a_cksum); if (xar_hash_check(a_hash, result, expected) != 0) { cli_dbgmsg("cli_scanxar: archived-checksum mismatch.\n"); cksum_fails++; } else { cli_dbgmsg("cli_scanxar: archived-checksum matched.\n"); } free(expected); } if (e_hash_ctx != NULL) { xar_hash_final(e_hash_ctx, result, e_hash); e_hash_ctx = NULL; } else { cli_dbgmsg("cli_scanxar: extracted-checksum(unarchived-checksum) missing.\n"); cksum_fails++; } if (e_cksum != NULL) { if (do_extract_cksum) { expected = cli_hex2str((char *)e_cksum); if (xar_hash_check(e_hash, result, expected) != 0) { cli_dbgmsg("cli_scanxar: extracted-checksum mismatch.\n"); cksum_fails++; } else { cli_dbgmsg("cli_scanxar: extracted-checksum matched.\n"); } free(expected); } } rc = cli_magic_scandesc(fd, ctx); if (rc != CL_SUCCESS) { if (rc == CL_VIRUS) { cli_dbgmsg("cli_scanxar: Infected with %s\n", cli_get_last_virus(ctx)); if (!SCAN_ALL) goto exit_tmpfile; } else if (rc != CL_BREAK) { cli_dbgmsg("cli_scanxar: cli_magic_scandesc error %i\n", rc); goto exit_tmpfile; } } } if (a_cksum != NULL) { xmlFree(a_cksum); a_cksum = NULL; } if (e_cksum != NULL) { xmlFree(e_cksum); e_cksum = NULL; } } exit_tmpfile: xar_cleanup_temp_file(ctx, fd, tmpname); if (a_hash_ctx != NULL) xar_hash_final(a_hash_ctx, result, a_hash); if (e_hash_ctx != NULL) xar_hash_final(e_hash_ctx, result, e_hash); exit_reader: if (a_cksum != NULL) xmlFree(a_cksum); if (e_cksum != NULL) xmlFree(e_cksum); xmlTextReaderClose(reader); xmlFreeTextReader(reader); exit_toc: free(toc); if (rc == CL_BREAK) rc = CL_SUCCESS; #else cli_dbgmsg("cli_scanxar: can't scan xar files, need libxml2.\n"); #endif if (cksum_fails + extract_errors != 0) { cli_warnmsg("cli_scanxar: %u checksum errors and %u extraction errors, use --debug for more info.\n", cksum_fails, extract_errors); } return rc; }
ParticlesDataMutable* readPRT(const char* filename,const bool headersOnly,std::ostream* errorStream) { std::auto_ptr<std::istream> input(new std::ifstream(filename,std::ios::in|std::ios::binary)); if (!*input) { if(errorStream) *errorStream<<"Partio: Unable to open file "<<filename<<std::endl; return 0; } // Use simple particle since we don't have optimized storage. ParticlesDataMutable* simple=0; if (headersOnly) simple=new ParticleHeaders; else simple=create(); FileHeadder header; input->read((char*)&header,sizeof(FileHeadder)); if (memcmp(header.magic, magic, sizeof(magic))) { if(errorStream) *errorStream<<"Partio: failed to get PRT magic"<<std::endl; return 0; } // The header may be a different size in other PRT versions if (header.headersize > sizeof(FileHeadder)) input->seekg(header.headersize); int reserve=0; int channels=0; int channelsize=0; read<LITEND>(*input,reserve); // reserved read<LITEND>(*input,channels); // number of channel read<LITEND>(*input,channelsize); // size of channel simple->addParticles((const int)header.numParticles); std::vector<Channel> chans; std::vector<ParticleAttribute> attrs; unsigned particleSize = 0; for (int i=0; i<channels; i++) { Channel ch; input->read((char*)&ch, sizeof(Channel)); ParticleAttributeType type=NONE; switch (ch.type) { case 0: // int16 case 1: // int32 case 2: // int64 type = INT; break; case 3: // float16 case 4: // float32 case 5: // float64 if (ch.arity == 3) type = VECTOR; else type = FLOAT; break; case 6: // uint16 case 7: // uint32 case 8: // uint64 type = INT; break; case 9: // int8 case 10:// uint8 type = INT; break; } if (type != NONE) { #ifdef AUTO_CASES if (ch.name[0] >= 'A' && ch.name[0] <= 'Z') { ch.name[0] += 0x20; } #endif std::string name((char*)ch.name); ParticleAttribute attrHandle=simple->addAttribute(name.c_str(),type,ch.arity); chans.push_back(ch); attrs.push_back(attrHandle); } // The size of the particle is determined from the channel with largest offset. The channels are not required to be listed in order. particleSize = (std::max)( particleSize, chans.back().offset + sizes[ch.type] ); // The channel entry might have more data in other PRT versions. if ((unsigned)channelsize > sizeof(Channel)) input->seekg(channelsize - sizeof(Channel), std::ios::cur); } if (headersOnly) return simple; z_stream z; z.zalloc = Z_NULL;z.zfree = Z_NULL;z.opaque = Z_NULL; if (inflateInit( &z ) != Z_OK) { if(errorStream) *errorStream<<"Zlib inflateInit error"<<std::endl; return 0; } char in_buf[OUT_BUFSIZE]; z.next_in = 0; z.avail_in = 0; char* prt_buf = new char[particleSize]; for (unsigned int particleIndex=0;particleIndex<(unsigned int )simple->numParticles();particleIndex++) { // Read the particle from the file, and decompress it into a single particle-sized buffer. read_buffer(*input, z, (char*)in_buf, prt_buf, particleSize, errorStream); for (unsigned int attrIndex=0;attrIndex<attrs.size();attrIndex++) { if (attrs[attrIndex].type==Partio::INT) { int* data=simple->dataWrite<int>(attrs[attrIndex],particleIndex); for (int count=0;count<attrs[attrIndex].count;count++) { int ival = 0; switch (chans[attrIndex].type) { case 0: // int16 { ival = (int)*reinterpret_cast<short*>( &prt_buf[ chans[attrIndex].offset + count * sizeof(short) ] ); } break; case 1: // int32 { ival = (int)*reinterpret_cast<int*>( &prt_buf[ chans[attrIndex].offset + count * sizeof(int) ] ); } break; case 2: // int64 { ival = (int)*reinterpret_cast<long long*>( &prt_buf[ chans[attrIndex].offset + count * sizeof(long long) ] ); } break; case 6: // uint16 { ival = (int)*reinterpret_cast<unsigned short*>( &prt_buf[ chans[attrIndex].offset + count * sizeof(unsigned short) ] ); } break; case 7: // uint32 { ival = (int)*reinterpret_cast<unsigned int*>( &prt_buf[ chans[attrIndex].offset + + count * sizeof(unsigned int) ] ); } break; case 8: // uint64 { ival = (int)*reinterpret_cast<unsigned long long*>( &prt_buf[ chans[attrIndex].offset + count * sizeof(unsigned long long) ] ); } break; case 9: // int8 { ival = (int)prt_buf[ chans[attrIndex].offset + count ]; } break; case 10:// uint8 { ival = (int)*reinterpret_cast<unsigned char*>( &prt_buf[ chans[attrIndex].offset + count * sizeof(unsigned char) ] ); } break; } data[count]=ival; } }else if (attrs[attrIndex].type==Partio::FLOAT || attrs[attrIndex].type==Partio::VECTOR) { float* data=simple->dataWrite<float>(attrs[attrIndex],particleIndex); for (int count=0;count<attrs[attrIndex].count;count++) { float fval = 0; switch (chans[attrIndex].type) { case 3: // float16 { #ifdef USE_ILMHALF fval = (float)*reinterpret_cast<half*>( &prt_buf[ chans[attrIndex].offset + count * sizeof(half) ] ); #else unsigned short val = *reinterpret_cast<unsigned short*>( &prt_buf[ chans[attrIndex].offset + count * sizeof(unsigned short) ] ); fval = half2float[val].f; #endif } break; case 4: // float32 { fval = (float)*reinterpret_cast<float*>( &prt_buf[ chans[attrIndex].offset + count * sizeof(float) ] ); } break; case 5: // float64 { fval = (float)*reinterpret_cast<double*>( &prt_buf[ chans[attrIndex].offset + count * sizeof(double) ] ); } break; } data[count]=fval; } } } } delete prt_buf; if (inflateEnd( &z ) != Z_OK) { if(errorStream) *errorStream<<"Zlib inflateEnd error"<<std::endl; return 0; } // success return simple; }
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF) { z_streamp zp; char *out; int res = Z_OK; size_t cnt; char *buf; size_t buflen; zp = (z_streamp) pg_malloc(sizeof(z_stream)); zp->zalloc = Z_NULL; zp->zfree = Z_NULL; zp->opaque = Z_NULL; buf = pg_malloc(ZLIB_IN_SIZE); buflen = ZLIB_IN_SIZE; out = pg_malloc(ZLIB_OUT_SIZE + 1); if (inflateInit(zp) != Z_OK) exit_horribly(modulename, "could not initialize compression library: %s\n", zp->msg); /* no minimal chunk size for zlib */ while ((cnt = readF(AH, &buf, &buflen))) { zp->next_in = (void *) buf; zp->avail_in = cnt; while (zp->avail_in > 0) { zp->next_out = (void *) out; zp->avail_out = ZLIB_OUT_SIZE; res = inflate(zp, 0); if (res != Z_OK && res != Z_STREAM_END) exit_horribly(modulename, "could not uncompress data: %s\n", zp->msg); out[ZLIB_OUT_SIZE - zp->avail_out] = '\0'; ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH); } } zp->next_in = NULL; zp->avail_in = 0; while (res != Z_STREAM_END) { zp->next_out = (void *) out; zp->avail_out = ZLIB_OUT_SIZE; res = inflate(zp, 0); if (res != Z_OK && res != Z_STREAM_END) exit_horribly(modulename, "could not uncompress data: %s\n", zp->msg); out[ZLIB_OUT_SIZE - zp->avail_out] = '\0'; ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH); } if (inflateEnd(zp) != Z_OK) exit_horribly(modulename, "could not close compression library: %s\n", zp->msg); free(buf); free(out); free(zp); }