static std::vector<inflater> build_inflaters() { std::vector<inflater> output; regex_t re; // gzip regcomp(&re, "\\.gz$", REG_EXTENDED); output.push_back(inflater(re, "gunzip -c '%s'")); // zip regcomp(&re, "\\.zip$", REG_EXTENDED); output.push_back(inflater(re, "unzip -p '%s'")); // bz2 regcomp(&re, "\\.bz2$", REG_EXTENDED); output.push_back(inflater(re, "bunzip2 -c '%s'")); // xz regcomp(&re, "\\.xz$", REG_EXTENDED); output.push_back(inflater(re, "unxz -c '%s'")); // lzma regcomp(&re, "\\.lzma$", REG_EXTENDED); output.push_back(inflater(re, "unlzma -c '%s'")); return output; }
// -------------------------------------------------------------------------- // -------------------------------------------------------------------------- bool t_single_scenario_campaign_file_header::read( std::streambuf & stream, t_progress_handler * progress_handler_ptr ) { m_map_data_start = stream.pubseekoff( 0, std::ios::cur ); m_map_data_end = stream.pubseekoff( 0, std::ios::end ); stream.pubseekpos( m_map_data_start ); try { t_inflate_filter inflater( stream ); if ( !::read( inflater, *m_map_header_ptr, progress_handler_ptr ) ) return false; } catch ( t_inflate_filter::t_data_error const & ) { return false; } return true; }
// -------------------------------------------------------------------------- // -------------------------------------------------------------------------- bool t_multi_scenario_campaign_file_header::read( std::streambuf & stream, t_progress_handler * progress_handler_ptr ) { int const k_current_format_version = 1; int format_version = get< t_uint16 >( stream ); if ( format_version < 0 || format_version > k_current_format_version ) return false; std::string new_name; std::string new_description; if ( !read_string16( stream, new_name ) || ( format_version >= 1 && !read_string16( stream, new_description ) ) ) return false; int map_count = get< t_uint16 >( stream ); if ( map_count <= 0 ) return false; std::vector< std::streambuf::off_type > map_data_sizes( map_count ); std::vector< t_map_info > new_map_info_vector( map_count ); int map_num; for ( map_num = 0; map_num < map_count; ++map_num ) map_data_sizes[ map_num ] = get< t_uint32 >( stream ); std::streambuf::pos_type start_pos = stream.pubseekoff( 0, std::ios::cur ); map_num = 0; while( true ) { std::streambuf::pos_type end_pos = start_pos + map_data_sizes[ map_num ]; t_map_info & map_info = new_map_info_vector[ map_num ]; map_info.data_start = start_pos; map_info.data_end = end_pos; map_info.header_ptr = new t_map_header; stream.pubseekpos( start_pos ); try { t_inflate_filter inflater( stream ); if ( !::read( inflater, *map_info.header_ptr, progress_handler_ptr ) ) return false; } catch ( t_inflate_filter::t_data_error const & ) { return false; } if ( ++map_num >= map_count ) break; start_pos = end_pos; } m_name.swap( new_name ); m_description.swap( new_description ); m_map_info_vector.swap( new_map_info_vector ); return true; }
bool zip_archive_impl::read_file_entry(const pstring& entry_name, vector<unsigned char>& buf) const { pstring name(entry_name); filename_map_type::const_iterator it = m_filenames.find(name); if (it == m_filenames.end()) // entry name not found. return false; size_t index = it->second; if (index >= m_file_params.size()) // entry index is out of bound. return false; const zip_file_param& param = m_file_params[index]; // Skip the file header section. zip_stream_parser file_header(m_stream, param.offset_file_header); file_header.skip_bytes(4); file_header.skip_bytes(2); file_header.skip_bytes(2); file_header.skip_bytes(2); file_header.skip_bytes(2); file_header.skip_bytes(2); file_header.skip_bytes(4); file_header.skip_bytes(4); file_header.skip_bytes(4); uint16_t filename_len = file_header.read_2bytes(); uint16_t extra_field_len = file_header.read_2bytes(); file_header.skip_bytes(filename_len); file_header.skip_bytes(extra_field_len); // Data section is immediately followed by the header section. m_stream->seek(file_header.tell()); vector<unsigned char> raw_buf(param.size_compressed+1, 0); m_stream->read(&raw_buf[0], param.size_compressed); switch (param.compress_method) { case zip_file_param::stored: // Not compressed at all. buf.swap(raw_buf); return true; case zip_file_param::deflated: { // deflate compression vector<unsigned char> zip_buf(param.size_uncompressed+1, 0); // null-terminated zip_inflater inflater(raw_buf, zip_buf, param); if (!inflater.init()) break; if (!inflater.inflate()) throw zip_error("error during inflate."); buf.swap(zip_buf); return true; } default: ; } return false; }
bool File::load(string path) { //load the blend file into the stream if(file.is_open()) file.close(); file.open(ofToDataPath(path, true).c_str(), ios::binary); //info should contain blender now, if not it is compressed string info = readString(7); //check if the file is gzipped if(info != "BLENDER") { seek(0); //unzip the blend file to a temp file and reload Poco::InflatingInputStream inflater(file, Poco::InflatingStreamBuf::STREAM_GZIP); Poco::TemporaryFile tempFile; tempFile.keepUntilExit(); std::ofstream out(tempFile.path().c_str(), ios::binary); Poco::StreamCopier::copyStream( inflater, out); out.close(); file.close(); file.open(tempFile.path().c_str(), ios::binary); info = readString(7); if(info != "BLENDER") { ofLogWarning(OFX_BLENDER) << "Could not read blend file " << path; } else { ofLogVerbose(OFX_BLENDER) << "Blend file is gzipped, temporarily decompressed contents to " << tempFile.path(); } } //now extract the rest of the header data string tempString = readString(1); if(tempString == "-") pointerSize = 8; else if(tempString == "_") pointerSize = 4; //ofLogVerbose(OFX_BLENDER) << "Pointer Size is " << pointerSize; if(pointerSize == 4) { readPointer = std::bind(&File::read<unsigned int>, this); } else { readPointer = std::bind(&File::read<unsigned long>, this); } bool littleEndianness; char structPre = ' '; tempString = readString(1); if(tempString == "v") { littleEndianness = true; structPre = '<'; } else if(tempString == "V") { littleEndianness = false; structPre = '>'; } //ofLogVerbose(OFX_BLENDER) << "Struct pre is " << structPre; //ofLogVerbose(OFX_BLENDER) << "Little Endianness is " << littleEndianness; //version version = readString(3); //now go through all them blocks blocks.push_back(Block(this)); //read the first block readHeader(blocks.back()); while(blocks.back().code != "DNA1" && blocks.back().code != "SDNA") { //skip the block data file.seekg(file.tellg() + streamoff(blocks.back().size)); //read a new block blocks.push_back(Block(this)); readHeader(blocks.back()); } //advance readString(4); readString(4); //NAMES unsigned int numNames = read<unsigned int>(); for(unsigned int i=0; i<numNames; i++) { catalog.names.push_back(DNAName(readString(0))); } align(file); //TYPES readString(4); unsigned int numTypes = read<unsigned int>(); //cout << "FOUND TYPES " << numTypes << endl; for(unsigned int i=0; i<numTypes; i++) { catalog.types.push_back(DNAType(readString(0), i)); } align(file); //TYPE LENGTHS readString(4);; for(unsigned int i=0; i<numTypes; i++) { catalog.types[i].size = read<unsigned short>(); if(catalog.types[i].size == 0) //assume it is a pointer catalog.types[i].size = pointerSize; } align(file); //STRUCTURES readString(4); unsigned int numStructs = read<unsigned int>(); //cout << "FOUND STRUCTURES " << numStructs << endl; for(unsigned int i=0; i<numStructs; i++) { //get the type unsigned int index = read<unsigned short>(); DNAType* type = &catalog.types[index]; catalog.structures.push_back(DNAStructure(type)); DNAStructure& structure = catalog.structures.back(); //get the fields for the structure unsigned short numFields = read<unsigned short>(); unsigned int curOffset = 0; for(unsigned int j=0; j<numFields; j++) { unsigned short typeIndex = read<unsigned short>(); unsigned short nameIndex = read<unsigned short>(); DNAType* type = &catalog.types[typeIndex]; DNAName* name = &catalog.names[nameIndex]; structure.fields.push_back(DNAField(type, name, curOffset)); //if the field is a pointer, then only add the pointer size to offset bool offsetSet = false; if(structure.fields.back().isPointer) { int amount = 0; if(structure.fields.back().isArray) { amount = structure.fields.back().arraySizes[0]; } if(amount == 0) amount = 1; curOffset += (pointerSize * amount); offsetSet = true; } else if(structure.fields.back().isArray) { //arrays add n times the size to offset float multi = 0; for(int s: structure.fields.back().arraySizes) { if(s!=-1) { if(multi == 0) multi += s; else multi *= s; } } if(multi != 0) offsetSet = true; curOffset += type->size * multi; } if(!offsetSet) { curOffset += type->size; } } } align(file); //now link all structures with the File Blocks vector<Block>::iterator it = blocks.begin(); while(it != blocks.end()) { (*it).structure = &catalog.structures[(*it).SDNAIndex]; it++; } ofLogVerbose(OFX_BLENDER) << "Loaded \"" << path << "\" - Blender version is " << version; return true; }
int readUncompressedBytes(FILE *fileIn, char *buffer, int bufsize) { compFile_t *cf = (compFile_t *)fileIn; compBlockHeader_t header; int bufferIndex = 0; SCP_LOG("readUncompressedBytes bufsize:%d", __FUNCTION__, bufsize); if (cf->leftOverBytes > 0) { SCP_LOG("LEFT OVER BYTES: %d bytes", __FUNCTION__, cf->leftOverBytes); if (cf->leftOverBytes >= bufsize) { SCP_LOG("COPYING %d BYTES TO BUFFER", __FUNCTION__, bufsize); memcpy(buffer, cf->uncompressedBytesArray, bufsize); cf->leftOverBytes -= bufsize; SCP_LOG("SLIDDING UP uncompressedBytesArray buffer from %d to 0.", __FUNCTION__, bufsize); memcpy(cf->uncompressedBytesArray, &cf->uncompressedBytesArray[bufsize], cf->leftOverBytes); SCP_LOG("RETURNING %d BYTES READ", __FUNCTION__, bufsize); return bufsize; } else { SCP_LOG("COPYING %d BYTES TO BUFFER", __FUNCTION__, cf->leftOverBytes); memcpy(buffer, cf->uncompressedBytesArray, cf->leftOverBytes); bufferIndex = cf->leftOverBytes; SCP_LOG("bufferIndex=%d", __FUNCTION__, bufferIndex); cf->leftOverBytes = 0; } } while((bufferIndex < bufsize) && !feof(cf->fdin) && !cf->endOfCompressedFile) { SCP_LOG("bufferIndex=%d", __FUNCTION__, bufferIndex); int bread = fread(&header, 1, sizeof(compBlockHeader_t), cf->fdin); if (bread != sizeof(compBlockHeader_t)) { SCP_LOG("Tried to read header of %d bytes and read %d!", __FUNCTION__, sizeof(compBlockHeader_t), bread); // should have got a tail header! return -1; } SCP_LOG("header.uncompressedBytes=%d header.compressedBytes=%d", __FUNCTION__, header.uncompressedBytes, header.compressedBytes); if ((header.uncompressedBytes == 0) && (header.compressedBytes == 0)) { // no more blocks cf->endOfCompressedFile = 1; SCP_LOG("Read tail header. EOF!", __FUNCTION__); break; } if ((header.uncompressedBytes > UNCOMPRESSED_ARRAY_SIZE) || (header.uncompressedBytes <= 0)) { SCP_LOG("Uncompressed bytes should be 1-%d and it's %d", __FUNCTION__, UNCOMPRESSED_ARRAY_SIZE, header.uncompressedBytes); return -1; } if ((header.compressedBytes > COMPRESSED_ARRAY_SIZE) || (header.compressedBytes <= 0)) { SCP_LOG("Compressed bytes should be 1-%d and it's %d", __FUNCTION__, COMPRESSED_ARRAY_SIZE, header.compressedBytes); return -1; } if (fread(cf->compressedBytesArray, 1, header.compressedBytes, cf->fdin) != header.compressedBytes) { SCP_LOG("Unable to read %d compressed bytes", __FUNCTION__, header.compressedBytes); return -1; } SCP_LOG("READ %d COMPRESSED BYTES", __FUNCTION__, header.compressedBytes); int bytesToCopy = 0; int uncompBytes = header.compressedBytes; if (header.compressedBytes < header.uncompressedBytes) { uncompBytes = inflater(cf->uncompressedBytesArray, UNCOMPRESSED_ARRAY_SIZE, cf->compressedBytesArray, header.compressedBytes); SCP_LOG("DECOMPRESSED %d BYTES INTO %d BYTES", __FUNCTION__, header.compressedBytes, uncompBytes); bytesToCopy = uncompBytes; } else { // this block is not compressed, just copy it memcpy(cf->uncompressedBytesArray, cf->compressedBytesArray, header.compressedBytes); bytesToCopy = header.compressedBytes; } if (uncompBytes + bufferIndex > bufsize) { // we have extra bytes. Save them for the next call. bytesToCopy = bufsize - bufferIndex; cf->leftOverBytes = uncompBytes - bytesToCopy; } SCP_LOG("COPYING %d BYTES FROM uncompressedBytesArray TO BUFFER[%d]", __FUNCTION__, bytesToCopy, bufferIndex); memcpy(&buffer[bufferIndex], cf->uncompressedBytesArray, bytesToCopy); if (cf->leftOverBytes > 0) { SCP_LOG("LEFT OVER BYTES. COPYING %d BYTES FROM %d TO 0.", __FUNCTION__, cf->leftOverBytes, bytesToCopy); memcpy(cf->uncompressedBytesArray, &cf->uncompressedBytesArray[bytesToCopy], cf->leftOverBytes); } bufferIndex += bytesToCopy; SCP_LOG("bufferIndex is now:%d", __FUNCTION__, bufferIndex); } SCP_LOG("EXIT: bufferIndex is now:%d", __FUNCTION__, bufferIndex); if (bufferIndex > 0 ) { if (bufferIndex < bufsize) { buffer[bufferIndex] = 0; } SCP_LOG("RETURNING: %s", __FUNCTION__, buffer); } return bufferIndex; }
int copyFile(char *src, char*dst, int compressFlag, long *compressedFileSize) { // compress = 1 means inflate // compress = 0 means just copy // compress = -1 means deflate time_t timeStamp; char *copyBuffer1 = NULL; char *copyBuffer2 = NULL; char *uncompressedBytesArray = NULL; char *compressedBytesArray = NULL; copyBuffer1 = malloc(COPYBUFFER_SIZE); copyBuffer2 = malloc(COPYBUFFER_SIZE); uncompressedBytesArray = malloc(UNCOMPRESSED_ARRAY_SIZE); compressedBytesArray = malloc(COMPRESSED_ARRAY_SIZE); if ((copyBuffer1 == NULL) || (copyBuffer2 == NULL) || (uncompressedBytesArray == NULL) || (compressedBytesArray == NULL)) { if (copyBuffer1 != NULL) free(copyBuffer1); if (copyBuffer2 != NULL) free(copyBuffer2); if (uncompressedBytesArray != NULL) free(uncompressedBytesArray); if (compressedBytesArray != NULL) free(compressedBytesArray); SCP_LOG("out of memory in copyFile", __FUNCTION__); } SCP_LOG("CopyFile src:%s, dst:%s, compressFlag:%d", __FUNCTION__, src, dst, compressFlag); size_t bufsize = COPYBUFFER_SIZE; int retVal = 0; int didFileHeader = 0; FILE *fdout; if (compressedFileSize != NULL) *compressedFileSize = 0; fdout = fopen(dst, "w"); // if we are out of space, try removing the original first and reopening the file if (fdout == NULL) { FileRemove(dst); fdout = fopen(dst, "w"); } if (fdout != NULL) { SCP_LOG("OPENED! output file %s", __FUNCTION__, dst); FILE *fdin; if (NULL != (fdin = fopen(src, "r"))) { SCP_LOG("OPENED! input %s", __FUNCTION__, src); while (!feof(fdin)) { int bytes = 0; if (compressFlag == 0) { bytes = fread((void*)copyBuffer1, 1, bufsize, fdin); SCP_LOG("READ %d bytes when tried %d bytes", __FUNCTION__, bytes, bufsize); } if (compressFlag < 0) { if (!didFileHeader) { compFileHeader_t header; if(UiUtil_GetLocalTime( &timeStamp) == 0){ header.creationTime = timeStamp; } else { header.creationTime = time(NULL); } header.version = COMP_VERSION; int bout = fwrite((void*)&header, 1, sizeof(compFileHeader_t), fdout); if (bout != sizeof(compFileHeader_t)) { SCP_LOG("Tried to write %d bytes to %s and only wrote %d", __FUNCTION__, sizeof(compFileHeader_t), dst, bout); retVal = -1; break; } else { SCP_LOG("Wrote file header to %s", __FUNCTION__, dst); } didFileHeader = 1; } bytes = fread((void*)copyBuffer1, 1, bufsize, fdin); SCP_LOG("READ %d bytes when tried %d bytes", __FUNCTION__, bytes, bufsize); int compLen = bytes; compress(copyBuffer2, &compLen, copyBuffer1, bytes); SCP_LOG("Compressed %d bytes into %d bytes", __FUNCTION__, bytes, compLen); if (bytes == compLen) { SCP_LOG("No compression needed for this block.", __FUNCTION__); memcpy(copyBuffer2, copyBuffer1, bytes); } compBlockHeader_t *header = (compBlockHeader_t*)copyBuffer1; header->compressedBytes = compLen; header->uncompressedBytes = bytes; char *startOfData = copyBuffer1+sizeof(compBlockHeader_t); memcpy(startOfData, copyBuffer2, compLen); bytes = sizeof(compBlockHeader_t)+compLen; if (compressedFileSize != NULL) { *compressedFileSize += (long)bytes;; } } if (compressFlag > 0) { if (!didFileHeader) { compFileHeader_t header; if (fread((void*)&header, 1, sizeof(compFileHeader_t), fdin) != sizeof(compFileHeader_t)) { SCP_LOG("Tried to READ %d bytes of header and didn't", __FUNCTION__, sizeof(compFileHeader_t)); break; } if (header.version != COMP_VERSION) { SCP_LOG("Unsuported compress file verison %d", __FUNCTION__, header.version); retVal = -1; break; } didFileHeader = 1; } compBlockHeader_t header; if (fread((void*)&header, 1, sizeof(compBlockHeader_t), fdin) != sizeof(compBlockHeader_t)) { SCP_LOG("Tried to READ %d bytes of header and didn't", __FUNCTION__, sizeof(compBlockHeader_t)); break; } SCP_LOG("READ %d bytes of header. %d compressed bytes and %d uncompressed bytes", __FUNCTION__, sizeof(compBlockHeader_t), header.compressedBytes, header.uncompressedBytes); if ((header.compressedBytes == 0) && (header.uncompressedBytes == 0)) { SCP_LOG("Tail header read. EOF!", __FUNCTION__); break; } if ((header.compressedBytes > COMPRESSED_ARRAY_SIZE) || (header.compressedBytes <= 0)) { SCP_LOG("Compressed bytes should be 0-%d and it's %d", __FUNCTION__, COMPRESSED_ARRAY_SIZE, header.compressedBytes); retVal = -1; break; } if ((header.uncompressedBytes > UNCOMPRESSED_ARRAY_SIZE) || (header.uncompressedBytes <= 0)) { SCP_LOG("Uncompressed bytes should be 1-%d and it's %d", __FUNCTION__, UNCOMPRESSED_ARRAY_SIZE, header.uncompressedBytes); retVal = -1; break; } if (fread(copyBuffer1, 1, header.compressedBytes, fdin) != header.compressedBytes) { SCP_LOG("Tried to READ %d bytes of data and didn't", __FUNCTION__, header.compressedBytes); retVal = -1; break; } SCP_LOG("READ %d bytes of data.", __FUNCTION__, header.compressedBytes); if (header.uncompressedBytes > header.compressedBytes) { SCP_LOG("About to decompress %d bytes", __FUNCTION__, header.compressedBytes); int uncompBytes = inflater(copyBuffer2, COPYBUFFER_SIZE, copyBuffer1, header.compressedBytes); SCP_LOG("Decompressed %d bytes into %d bytes", __FUNCTION__, header.compressedBytes, uncompBytes); memcpy(copyBuffer1, copyBuffer2, uncompBytes); SCP_LOG("DECOMP:%2048s", __FUNCTION__, dst); bytes = uncompBytes; } else { SCP_LOG("NO DECOMP NEEDED.", __FUNCTION__); bytes = header.compressedBytes; } } if (bytes > 0) { SCP_LOG("Writing %d bytes to %s", __FUNCTION__, bytes, dst); int bout = fwrite((void*)copyBuffer1, 1, bytes, fdout); if (bout != bytes) { SCP_LOG("Tried to write %d bytes to %s and only wrote %d", __FUNCTION__, bytes, dst, bout); retVal = -1; break; } else { SCP_LOG("Wrote %d bytes to %s", __FUNCTION__, bytes, dst); } } else {break;} } if (compressFlag < 0) { compBlockHeader_t tailHeader; tailHeader.compressedBytes = 0; tailHeader.uncompressedBytes = 0; int bout = fwrite((void*)&tailHeader, 1, sizeof(compBlockHeader_t), fdout); if (bout != sizeof(compBlockHeader_t)) { SCP_LOG("Tried to write tail header to %s and only wrote %d", __FUNCTION__, dst, bout); retVal = -1; } else { if (compressedFileSize != NULL) { *compressedFileSize += (long)bout; } SCP_LOG("Wrote tail header to %s", __FUNCTION__, dst); } } fclose(fdin); SCP_LOG("Closed fdin", __FUNCTION__); } else { SCP_LOG("Unable to fopen input file %s", __FUNCTION__, src); retVal = -1; } fclose(fdout); SCP_LOG("Closed fdout", __FUNCTION__); } else { SCP_LOG("Unable to fopen %s as writeable.", __FUNCTION__, dst); retVal = -1; } SCP_LOG("CopyFile returning %d", __FUNCTION__, retVal); free(compressedBytesArray); free(uncompressedBytesArray); free(copyBuffer2); free(copyBuffer1); return retVal; }