static std::vector<inflater> build_inflaters()
{
std::vector<inflater> output;
regex_t re;
// gzip
regcomp(&re, "\\.gz$", REG_EXTENDED);
output.push_back(inflater(re, "gunzip -c '%s'"));
// zip
regcomp(&re, "\\.zip$", REG_EXTENDED);
output.push_back(inflater(re, "unzip -p '%s'"));
// bz2
regcomp(&re, "\\.bz2$", REG_EXTENDED);
output.push_back(inflater(re, "bunzip2 -c '%s'"));
// xz
regcomp(&re, "\\.xz$", REG_EXTENDED);
output.push_back(inflater(re, "unxz -c '%s'"));
// lzma
regcomp(&re, "\\.lzma$", REG_EXTENDED);
output.push_back(inflater(re, "unlzma -c '%s'"));
return output;
}
// --------------------------------------------------------------------------
// --------------------------------------------------------------------------
bool t_single_scenario_campaign_file_header::read(
std::streambuf & stream,
t_progress_handler * progress_handler_ptr )
{
m_map_data_start = stream.pubseekoff( 0, std::ios::cur );
m_map_data_end = stream.pubseekoff( 0, std::ios::end );
stream.pubseekpos( m_map_data_start );
try
{
t_inflate_filter inflater( stream );
if ( !::read( inflater, *m_map_header_ptr, progress_handler_ptr ) )
return false;
}
catch ( t_inflate_filter::t_data_error const & )
{
return false;
}
return true;
}
// --------------------------------------------------------------------------
// --------------------------------------------------------------------------
bool t_multi_scenario_campaign_file_header::read(
std::streambuf & stream,
t_progress_handler * progress_handler_ptr )
{
int const k_current_format_version = 1;
int format_version = get< t_uint16 >( stream );
if ( format_version < 0 || format_version > k_current_format_version )
return false;
std::string new_name;
std::string new_description;
if ( !read_string16( stream, new_name )
|| ( format_version >= 1 && !read_string16( stream, new_description ) ) )
return false;
int map_count = get< t_uint16 >( stream );
if ( map_count <= 0 )
return false;
std::vector< std::streambuf::off_type > map_data_sizes( map_count );
std::vector< t_map_info > new_map_info_vector( map_count );
int map_num;
for ( map_num = 0; map_num < map_count; ++map_num )
map_data_sizes[ map_num ] = get< t_uint32 >( stream );
std::streambuf::pos_type start_pos = stream.pubseekoff( 0, std::ios::cur );
map_num = 0;
while( true )
{
std::streambuf::pos_type end_pos = start_pos + map_data_sizes[ map_num ];
t_map_info & map_info = new_map_info_vector[ map_num ];
map_info.data_start = start_pos;
map_info.data_end = end_pos;
map_info.header_ptr = new t_map_header;
stream.pubseekpos( start_pos );
try
{
t_inflate_filter inflater( stream );
if ( !::read( inflater, *map_info.header_ptr, progress_handler_ptr ) )
return false;
}
catch ( t_inflate_filter::t_data_error const & )
{
return false;
}
if ( ++map_num >= map_count )
break;
start_pos = end_pos;
}
m_name.swap( new_name );
m_description.swap( new_description );
m_map_info_vector.swap( new_map_info_vector );
return true;
}
bool zip_archive_impl::read_file_entry(const pstring& entry_name, vector<unsigned char>& buf) const
{
pstring name(entry_name);
filename_map_type::const_iterator it = m_filenames.find(name);
if (it == m_filenames.end())
// entry name not found.
return false;
size_t index = it->second;
if (index >= m_file_params.size())
// entry index is out of bound.
return false;
const zip_file_param& param = m_file_params[index];
// Skip the file header section.
zip_stream_parser file_header(m_stream, param.offset_file_header);
file_header.skip_bytes(4);
file_header.skip_bytes(2);
file_header.skip_bytes(2);
file_header.skip_bytes(2);
file_header.skip_bytes(2);
file_header.skip_bytes(2);
file_header.skip_bytes(4);
file_header.skip_bytes(4);
file_header.skip_bytes(4);
uint16_t filename_len = file_header.read_2bytes();
uint16_t extra_field_len = file_header.read_2bytes();
file_header.skip_bytes(filename_len);
file_header.skip_bytes(extra_field_len);
// Data section is immediately followed by the header section.
m_stream->seek(file_header.tell());
vector<unsigned char> raw_buf(param.size_compressed+1, 0);
m_stream->read(&raw_buf[0], param.size_compressed);
switch (param.compress_method)
{
case zip_file_param::stored:
// Not compressed at all.
buf.swap(raw_buf);
return true;
case zip_file_param::deflated:
{
// deflate compression
vector<unsigned char> zip_buf(param.size_uncompressed+1, 0); // null-terminated
zip_inflater inflater(raw_buf, zip_buf, param);
if (!inflater.init())
break;
if (!inflater.inflate())
throw zip_error("error during inflate.");
buf.swap(zip_buf);
return true;
}
default:
;
}
return false;
}
bool File::load(string path) {
//load the blend file into the stream
if(file.is_open())
file.close();
file.open(ofToDataPath(path, true).c_str(), ios::binary);
//info should contain blender now, if not it is compressed
string info = readString(7);
//check if the file is gzipped
if(info != "BLENDER") {
seek(0);
//unzip the blend file to a temp file and reload
Poco::InflatingInputStream inflater(file, Poco::InflatingStreamBuf::STREAM_GZIP);
Poco::TemporaryFile tempFile;
tempFile.keepUntilExit();
std::ofstream out(tempFile.path().c_str(), ios::binary);
Poco::StreamCopier::copyStream( inflater, out);
out.close();
file.close();
file.open(tempFile.path().c_str(), ios::binary);
info = readString(7);
if(info != "BLENDER") {
ofLogWarning(OFX_BLENDER) << "Could not read blend file " << path;
} else {
ofLogVerbose(OFX_BLENDER) << "Blend file is gzipped, temporarily decompressed contents to " << tempFile.path();
}
}
//now extract the rest of the header data
string tempString = readString(1);
if(tempString == "-")
pointerSize = 8;
else if(tempString == "_")
pointerSize = 4;
//ofLogVerbose(OFX_BLENDER) << "Pointer Size is " << pointerSize;
if(pointerSize == 4) {
readPointer = std::bind(&File::read<unsigned int>, this);
} else {
readPointer = std::bind(&File::read<unsigned long>, this);
}
bool littleEndianness;
char structPre = ' ';
tempString = readString(1);
if(tempString == "v") {
littleEndianness = true;
structPre = '<';
} else if(tempString == "V") {
littleEndianness = false;
structPre = '>';
}
//ofLogVerbose(OFX_BLENDER) << "Struct pre is " << structPre;
//ofLogVerbose(OFX_BLENDER) << "Little Endianness is " << littleEndianness;
//version
version = readString(3);
//now go through all them blocks
blocks.push_back(Block(this));
//read the first block
readHeader(blocks.back());
while(blocks.back().code != "DNA1" && blocks.back().code != "SDNA") {
//skip the block data
file.seekg(file.tellg() + streamoff(blocks.back().size));
//read a new block
blocks.push_back(Block(this));
readHeader(blocks.back());
}
//advance
readString(4);
readString(4);
//NAMES
unsigned int numNames = read<unsigned int>();
for(unsigned int i=0; i<numNames; i++) {
catalog.names.push_back(DNAName(readString(0)));
}
align(file);
//TYPES
readString(4);
unsigned int numTypes = read<unsigned int>();
//cout << "FOUND TYPES " << numTypes << endl;
for(unsigned int i=0; i<numTypes; i++) {
catalog.types.push_back(DNAType(readString(0), i));
}
align(file);
//TYPE LENGTHS
readString(4);;
for(unsigned int i=0; i<numTypes; i++) {
catalog.types[i].size = read<unsigned short>();
if(catalog.types[i].size == 0) //assume it is a pointer
catalog.types[i].size = pointerSize;
}
align(file);
//STRUCTURES
readString(4);
unsigned int numStructs = read<unsigned int>();
//cout << "FOUND STRUCTURES " << numStructs << endl;
for(unsigned int i=0; i<numStructs; i++) {
//get the type
unsigned int index = read<unsigned short>();
DNAType* type = &catalog.types[index];
catalog.structures.push_back(DNAStructure(type));
DNAStructure& structure = catalog.structures.back();
//get the fields for the structure
unsigned short numFields = read<unsigned short>();
unsigned int curOffset = 0;
for(unsigned int j=0; j<numFields; j++) {
unsigned short typeIndex = read<unsigned short>();
unsigned short nameIndex = read<unsigned short>();
DNAType* type = &catalog.types[typeIndex];
DNAName* name = &catalog.names[nameIndex];
structure.fields.push_back(DNAField(type, name, curOffset));
//if the field is a pointer, then only add the pointer size to offset
bool offsetSet = false;
if(structure.fields.back().isPointer) {
int amount = 0;
if(structure.fields.back().isArray) {
amount = structure.fields.back().arraySizes[0];
}
if(amount == 0)
amount = 1;
curOffset += (pointerSize * amount);
offsetSet = true;
} else if(structure.fields.back().isArray) { //arrays add n times the size to offset
float multi = 0;
for(int s: structure.fields.back().arraySizes) {
if(s!=-1) {
if(multi == 0)
multi += s;
else
multi *= s;
}
}
if(multi != 0)
offsetSet = true;
curOffset += type->size * multi;
}
if(!offsetSet) {
curOffset += type->size;
}
}
}
align(file);
//now link all structures with the File Blocks
vector<Block>::iterator it = blocks.begin();
while(it != blocks.end()) {
(*it).structure = &catalog.structures[(*it).SDNAIndex];
it++;
}
ofLogVerbose(OFX_BLENDER) << "Loaded \"" << path << "\" - Blender version is " << version;
return true;
}
int readUncompressedBytes(FILE *fileIn, char *buffer, int bufsize) {
compFile_t *cf = (compFile_t *)fileIn;
compBlockHeader_t header;
int bufferIndex = 0;
SCP_LOG("readUncompressedBytes bufsize:%d", __FUNCTION__, bufsize);
if (cf->leftOverBytes > 0) {
SCP_LOG("LEFT OVER BYTES: %d bytes", __FUNCTION__, cf->leftOverBytes);
if (cf->leftOverBytes >= bufsize) {
SCP_LOG("COPYING %d BYTES TO BUFFER", __FUNCTION__, bufsize);
memcpy(buffer, cf->uncompressedBytesArray, bufsize);
cf->leftOverBytes -= bufsize;
SCP_LOG("SLIDDING UP uncompressedBytesArray buffer from %d to 0.", __FUNCTION__, bufsize);
memcpy(cf->uncompressedBytesArray, &cf->uncompressedBytesArray[bufsize], cf->leftOverBytes);
SCP_LOG("RETURNING %d BYTES READ", __FUNCTION__, bufsize);
return bufsize;
} else {
SCP_LOG("COPYING %d BYTES TO BUFFER", __FUNCTION__, cf->leftOverBytes);
memcpy(buffer, cf->uncompressedBytesArray, cf->leftOverBytes);
bufferIndex = cf->leftOverBytes;
SCP_LOG("bufferIndex=%d", __FUNCTION__, bufferIndex);
cf->leftOverBytes = 0;
}
}
while((bufferIndex < bufsize) && !feof(cf->fdin) && !cf->endOfCompressedFile) {
SCP_LOG("bufferIndex=%d", __FUNCTION__, bufferIndex);
int bread = fread(&header, 1, sizeof(compBlockHeader_t), cf->fdin);
if (bread != sizeof(compBlockHeader_t)) {
SCP_LOG("Tried to read header of %d bytes and read %d!", __FUNCTION__, sizeof(compBlockHeader_t), bread);
// should have got a tail header!
return -1;
}
SCP_LOG("header.uncompressedBytes=%d header.compressedBytes=%d", __FUNCTION__, header.uncompressedBytes, header.compressedBytes);
if ((header.uncompressedBytes == 0) && (header.compressedBytes == 0)) {
// no more blocks
cf->endOfCompressedFile = 1;
SCP_LOG("Read tail header. EOF!", __FUNCTION__);
break;
}
if ((header.uncompressedBytes > UNCOMPRESSED_ARRAY_SIZE) || (header.uncompressedBytes <= 0)) {
SCP_LOG("Uncompressed bytes should be 1-%d and it's %d", __FUNCTION__, UNCOMPRESSED_ARRAY_SIZE, header.uncompressedBytes);
return -1;
}
if ((header.compressedBytes > COMPRESSED_ARRAY_SIZE) || (header.compressedBytes <= 0)) {
SCP_LOG("Compressed bytes should be 1-%d and it's %d", __FUNCTION__, COMPRESSED_ARRAY_SIZE, header.compressedBytes);
return -1;
}
if (fread(cf->compressedBytesArray, 1, header.compressedBytes, cf->fdin) != header.compressedBytes) {
SCP_LOG("Unable to read %d compressed bytes", __FUNCTION__, header.compressedBytes);
return -1;
}
SCP_LOG("READ %d COMPRESSED BYTES", __FUNCTION__, header.compressedBytes);
int bytesToCopy = 0;
int uncompBytes = header.compressedBytes;
if (header.compressedBytes < header.uncompressedBytes) {
uncompBytes = inflater(cf->uncompressedBytesArray, UNCOMPRESSED_ARRAY_SIZE,
cf->compressedBytesArray, header.compressedBytes);
SCP_LOG("DECOMPRESSED %d BYTES INTO %d BYTES", __FUNCTION__, header.compressedBytes, uncompBytes);
bytesToCopy = uncompBytes;
} else {
// this block is not compressed, just copy it
memcpy(cf->uncompressedBytesArray, cf->compressedBytesArray, header.compressedBytes);
bytesToCopy = header.compressedBytes;
}
if (uncompBytes + bufferIndex > bufsize) {
// we have extra bytes. Save them for the next call.
bytesToCopy = bufsize - bufferIndex;
cf->leftOverBytes = uncompBytes - bytesToCopy;
}
SCP_LOG("COPYING %d BYTES FROM uncompressedBytesArray TO BUFFER[%d]", __FUNCTION__, bytesToCopy, bufferIndex);
memcpy(&buffer[bufferIndex], cf->uncompressedBytesArray, bytesToCopy);
if (cf->leftOverBytes > 0) {
SCP_LOG("LEFT OVER BYTES. COPYING %d BYTES FROM %d TO 0.", __FUNCTION__, cf->leftOverBytes, bytesToCopy);
memcpy(cf->uncompressedBytesArray, &cf->uncompressedBytesArray[bytesToCopy], cf->leftOverBytes);
}
bufferIndex += bytesToCopy;
SCP_LOG("bufferIndex is now:%d", __FUNCTION__, bufferIndex);
}
SCP_LOG("EXIT: bufferIndex is now:%d", __FUNCTION__, bufferIndex);
if (bufferIndex > 0 ) {
if (bufferIndex < bufsize) {
buffer[bufferIndex] = 0;
}
SCP_LOG("RETURNING: %s", __FUNCTION__, buffer);
}
return bufferIndex;
}
int copyFile(char *src, char*dst, int compressFlag, long *compressedFileSize) {
// compress = 1 means inflate
// compress = 0 means just copy
// compress = -1 means deflate
time_t timeStamp;
char *copyBuffer1 = NULL;
char *copyBuffer2 = NULL;
char *uncompressedBytesArray = NULL;
char *compressedBytesArray = NULL;
copyBuffer1 = malloc(COPYBUFFER_SIZE);
copyBuffer2 = malloc(COPYBUFFER_SIZE);
uncompressedBytesArray = malloc(UNCOMPRESSED_ARRAY_SIZE);
compressedBytesArray = malloc(COMPRESSED_ARRAY_SIZE);
if ((copyBuffer1 == NULL) || (copyBuffer2 == NULL) ||
(uncompressedBytesArray == NULL) || (compressedBytesArray == NULL)) {
if (copyBuffer1 != NULL)
free(copyBuffer1);
if (copyBuffer2 != NULL)
free(copyBuffer2);
if (uncompressedBytesArray != NULL)
free(uncompressedBytesArray);
if (compressedBytesArray != NULL)
free(compressedBytesArray);
SCP_LOG("out of memory in copyFile", __FUNCTION__);
}
SCP_LOG("CopyFile src:%s, dst:%s, compressFlag:%d", __FUNCTION__, src, dst, compressFlag);
size_t bufsize = COPYBUFFER_SIZE;
int retVal = 0;
int didFileHeader = 0;
FILE *fdout;
if (compressedFileSize != NULL)
*compressedFileSize = 0;
fdout = fopen(dst, "w");
// if we are out of space, try removing the original first and reopening the file
if (fdout == NULL) {
FileRemove(dst);
fdout = fopen(dst, "w");
}
if (fdout != NULL) {
SCP_LOG("OPENED! output file %s", __FUNCTION__, dst);
FILE *fdin;
if (NULL != (fdin = fopen(src, "r"))) {
SCP_LOG("OPENED! input %s", __FUNCTION__, src);
while (!feof(fdin)) {
int bytes = 0;
if (compressFlag == 0) {
bytes = fread((void*)copyBuffer1, 1, bufsize, fdin);
SCP_LOG("READ %d bytes when tried %d bytes", __FUNCTION__, bytes, bufsize);
}
if (compressFlag < 0) {
if (!didFileHeader) {
compFileHeader_t header;
if(UiUtil_GetLocalTime( &timeStamp) == 0){
header.creationTime = timeStamp;
}
else {
header.creationTime = time(NULL);
}
header.version = COMP_VERSION;
int bout = fwrite((void*)&header, 1, sizeof(compFileHeader_t), fdout);
if (bout != sizeof(compFileHeader_t)) {
SCP_LOG("Tried to write %d bytes to %s and only wrote %d", __FUNCTION__, sizeof(compFileHeader_t), dst, bout);
retVal = -1;
break;
} else {
SCP_LOG("Wrote file header to %s", __FUNCTION__, dst);
}
didFileHeader = 1;
}
bytes = fread((void*)copyBuffer1, 1, bufsize, fdin);
SCP_LOG("READ %d bytes when tried %d bytes", __FUNCTION__, bytes, bufsize);
int compLen = bytes;
compress(copyBuffer2, &compLen, copyBuffer1, bytes);
SCP_LOG("Compressed %d bytes into %d bytes", __FUNCTION__, bytes, compLen);
if (bytes == compLen) {
SCP_LOG("No compression needed for this block.", __FUNCTION__);
memcpy(copyBuffer2, copyBuffer1, bytes);
}
compBlockHeader_t *header = (compBlockHeader_t*)copyBuffer1;
header->compressedBytes = compLen;
header->uncompressedBytes = bytes;
char *startOfData = copyBuffer1+sizeof(compBlockHeader_t);
memcpy(startOfData, copyBuffer2, compLen);
bytes = sizeof(compBlockHeader_t)+compLen;
if (compressedFileSize != NULL) {
*compressedFileSize += (long)bytes;;
}
}
if (compressFlag > 0) {
if (!didFileHeader) {
compFileHeader_t header;
if (fread((void*)&header, 1, sizeof(compFileHeader_t), fdin) != sizeof(compFileHeader_t)) {
SCP_LOG("Tried to READ %d bytes of header and didn't", __FUNCTION__, sizeof(compFileHeader_t));
break;
}
if (header.version != COMP_VERSION) {
SCP_LOG("Unsuported compress file verison %d", __FUNCTION__, header.version);
retVal = -1;
break;
}
didFileHeader = 1;
}
compBlockHeader_t header;
if (fread((void*)&header, 1, sizeof(compBlockHeader_t), fdin) != sizeof(compBlockHeader_t)) {
SCP_LOG("Tried to READ %d bytes of header and didn't", __FUNCTION__, sizeof(compBlockHeader_t));
break;
}
SCP_LOG("READ %d bytes of header. %d compressed bytes and %d uncompressed bytes", __FUNCTION__, sizeof(compBlockHeader_t), header.compressedBytes, header.uncompressedBytes);
if ((header.compressedBytes == 0) && (header.uncompressedBytes == 0)) {
SCP_LOG("Tail header read. EOF!", __FUNCTION__);
break;
}
if ((header.compressedBytes > COMPRESSED_ARRAY_SIZE) || (header.compressedBytes <= 0)) {
SCP_LOG("Compressed bytes should be 0-%d and it's %d", __FUNCTION__, COMPRESSED_ARRAY_SIZE, header.compressedBytes);
retVal = -1;
break;
}
if ((header.uncompressedBytes > UNCOMPRESSED_ARRAY_SIZE) || (header.uncompressedBytes <= 0)) {
SCP_LOG("Uncompressed bytes should be 1-%d and it's %d", __FUNCTION__, UNCOMPRESSED_ARRAY_SIZE, header.uncompressedBytes);
retVal = -1;
break;
}
if (fread(copyBuffer1, 1, header.compressedBytes, fdin) != header.compressedBytes) {
SCP_LOG("Tried to READ %d bytes of data and didn't", __FUNCTION__, header.compressedBytes);
retVal = -1;
break;
}
SCP_LOG("READ %d bytes of data.", __FUNCTION__, header.compressedBytes);
if (header.uncompressedBytes > header.compressedBytes) {
SCP_LOG("About to decompress %d bytes", __FUNCTION__, header.compressedBytes);
int uncompBytes = inflater(copyBuffer2, COPYBUFFER_SIZE, copyBuffer1, header.compressedBytes);
SCP_LOG("Decompressed %d bytes into %d bytes", __FUNCTION__, header.compressedBytes, uncompBytes);
memcpy(copyBuffer1, copyBuffer2, uncompBytes);
SCP_LOG("DECOMP:%2048s", __FUNCTION__, dst);
bytes = uncompBytes;
} else {
SCP_LOG("NO DECOMP NEEDED.", __FUNCTION__);
bytes = header.compressedBytes;
}
}
if (bytes > 0) {
SCP_LOG("Writing %d bytes to %s", __FUNCTION__, bytes, dst);
int bout = fwrite((void*)copyBuffer1, 1, bytes, fdout);
if (bout != bytes) {
SCP_LOG("Tried to write %d bytes to %s and only wrote %d", __FUNCTION__, bytes, dst, bout);
retVal = -1;
break;
} else {
SCP_LOG("Wrote %d bytes to %s", __FUNCTION__, bytes, dst);
}
} else {break;}
}
if (compressFlag < 0) {
compBlockHeader_t tailHeader;
tailHeader.compressedBytes = 0;
tailHeader.uncompressedBytes = 0;
int bout = fwrite((void*)&tailHeader, 1, sizeof(compBlockHeader_t), fdout);
if (bout != sizeof(compBlockHeader_t)) {
SCP_LOG("Tried to write tail header to %s and only wrote %d", __FUNCTION__, dst, bout);
retVal = -1;
} else {
if (compressedFileSize != NULL) {
*compressedFileSize += (long)bout;
}
SCP_LOG("Wrote tail header to %s", __FUNCTION__, dst);
}
}
fclose(fdin);
SCP_LOG("Closed fdin", __FUNCTION__);
} else {
SCP_LOG("Unable to fopen input file %s", __FUNCTION__, src);
retVal = -1;
}
fclose(fdout);
SCP_LOG("Closed fdout", __FUNCTION__);
} else {
SCP_LOG("Unable to fopen %s as writeable.", __FUNCTION__, dst);
retVal = -1;
}
SCP_LOG("CopyFile returning %d", __FUNCTION__, retVal);
free(compressedBytesArray);
free(uncompressedBytesArray);
free(copyBuffer2);
free(copyBuffer1);
return retVal;
}
