void test_strings()
{
// test unique
char str1[] = "gurkans";
char str2[] = "bananarama";
assert(has_unique_chars(str1) && !has_unique_chars(str2));
reverse_string(str1); reverse_string(str1);
// test are permutations
assert(are_permutations("snbba","snabb") && !are_permutations("snabb", "snbbb") &&
!are_permutations("snabb", "sbnb"));
// Test compression
char* str3 = "katt";
char* str4 = "kattttttttttttttttis";
assert(compressed(str3) == str3 && strlen(compressed(str4)) < strlen(str4));
// Test substrings
assert(issubstr("katt", "srkatten") && issubstr("katt", "katt") &&
!issubstr("katt", "kat") && issubstr("katt", "katten") &&
issubstr("katten", "katten"));
// Test rotation
assert(is_rotated("waterbottle", "erbottlewat") &&
!is_rotated("waterbottle", "bottleerwat") &&
is_rotated("waterbottle", "bottlewater"));
}
QVariant readBinaryArray(QDataStream& in, int& position) {
quint32 arrayLength;
quint32 encoding;
quint32 compressedLength;
in >> arrayLength;
in >> encoding;
in >> compressedLength;
position += sizeof(quint32) * 3;
QVector<T> values;
if ((int)QSysInfo::ByteOrder == (int)in.byteOrder()) {
values.resize(arrayLength);
QByteArray arrayData;
if (encoding == FBX_PROPERTY_COMPRESSED_FLAG) {
// preface encoded data with uncompressed length
QByteArray compressed(sizeof(quint32) + compressedLength, 0);
*((quint32*)compressed.data()) = qToBigEndian<quint32>(arrayLength * sizeof(T));
in.readRawData(compressed.data() + sizeof(quint32), compressedLength);
position += compressedLength;
arrayData = qUncompress(compressed);
if (arrayData.isEmpty() ||
(unsigned int)arrayData.size() != (sizeof(T) * arrayLength)) { // answers empty byte array if corrupt
throw QString("corrupt fbx file");
}
} else {
arrayData.resize(sizeof(T) * arrayLength);
position += sizeof(T) * arrayLength;
in.readRawData(arrayData.data(), arrayData.size());
}
if (arrayData.size() > 0) {
memcpy(&values[0], arrayData.constData(), arrayData.size());
}
} else {
values.reserve(arrayLength);
if (encoding == FBX_PROPERTY_COMPRESSED_FLAG) {
// preface encoded data with uncompressed length
QByteArray compressed(sizeof(quint32) + compressedLength, 0);
*((quint32*)compressed.data()) = qToBigEndian<quint32>(arrayLength * sizeof(T));
in.readRawData(compressed.data() + sizeof(quint32), compressedLength);
position += compressedLength;
QByteArray uncompressed = qUncompress(compressed);
if (uncompressed.isEmpty()) { // answers empty byte array if corrupt
throw QString("corrupt fbx file");
}
QDataStream uncompressedIn(uncompressed);
uncompressedIn.setByteOrder(QDataStream::LittleEndian);
uncompressedIn.setVersion(QDataStream::Qt_4_5); // for single/double precision switch
for (quint32 i = 0; i < arrayLength; i++) {
T value;
uncompressedIn >> value;
values.append(value);
}
} else {
for (quint32 i = 0; i < arrayLength; i++) {
void MapChunk::saveToFile(const String& fileName) {
std::stringstream strm;
strm.write((const char*) &mIndexX, sizeof(uint32));
strm.write((const char*) &mIndexY, sizeof(uint32));
for (uint32 y = 0; y < 16; ++y) {
for (uint32 x = 0; x < 16; ++x) {
auto& row = mRows[y * 16 + x];
uint16 height = (uint16) row.getHeight();
strm.write((const char*) &height, sizeof(uint16));
BlockType curBlock = BlockType::Invalid;
uint16 curCount = 0;
bool isFirst = true;
for (uint16 i = 0; i < height; ++i) {
++curCount;
if (isFirst == true) {
curBlock = row.getType(i);
isFirst = false;
continue;
}
if (i == (height - 1) || row.getType(i) != row.getType(i - 1)) {
strm.write((const char*) &curCount, sizeof(uint16));
strm.write((const char*) &curBlock, sizeof(uint16));
curCount = 0;
curBlock = row.getType(i);
}
}
if (curCount != 0) {
strm.write((const char*) &curCount, sizeof(uint16));
strm.write((const char*) &curBlock, sizeof(uint16));
}
}
}
uint32 size = (uint32)strm.tellp();
strm.seekp(0, std::ios::beg);
strm.seekg(0, std::ios::beg);
std::vector<char> compressed(size);
std::vector<char> uncompressed(size);
strm.read(uncompressed.data(), size);
Utils::ZDeflater deflater;
deflater.begin();
uint32 out = 0;
deflater.update(uncompressed, compressed, out);
deflater.end();
std::ofstream fileOut(fileName.toMultibyte(), std::ios::binary | std::ios::out);
fileOut.write((const char*) &size, sizeof(uint32));
fileOut.write((const char*) &out, sizeof(uint32));
fileOut.write(compressed.data(), out);
fileOut.close();
}
bool
MP4Reader::SkipVideoDemuxToNextKeyFrame(int64_t aTimeThreshold, uint32_t& parsed)
{
MOZ_ASSERT(GetTaskQueue()->IsCurrentThreadIn());
MOZ_ASSERT(mVideo.mDecoder);
Flush(kVideo);
// Loop until we reach the next keyframe after the threshold.
while (true) {
nsAutoPtr<MediaSample> compressed(PopSample(kVideo));
if (!compressed) {
// EOS, or error. This code assumes EOS, which may or may not be right.
MonitorAutoLock mon(mVideo.mMonitor);
mVideo.mDemuxEOS = true;
return false;
}
parsed++;
if (!compressed->mMp4Sample->is_sync_point ||
compressed->mMp4Sample->composition_timestamp < aTimeThreshold) {
continue;
}
mQueuedVideoSample = compressed;
break;
}
return true;
}
void SkPDFStream::emitObject(SkWStream* stream,
const SkPDFObjNumMap& objNumMap,
const SkPDFSubstituteMap& substitutes) {
if (fState == kUnused_State) {
fState = kNoCompression_State;
SkDynamicMemoryWStream compressedData;
SkAssertResult(
SkFlate::Deflate(fDataStream.get(), &compressedData));
SkAssertResult(fDataStream->rewind());
if (compressedData.getOffset() < this->dataSize()) {
SkAutoTDelete<SkStream> compressed(
compressedData.detachAsStream());
this->setData(compressed.get());
this->insertName("Filter", "FlateDecode");
}
fState = kCompressed_State;
this->insertInt("Length", this->dataSize());
}
this->INHERITED::emitObject(stream, objNumMap, substitutes);
stream->writeText(" stream\n");
stream->writeStream(fDataStream.get(), fDataStream->getLength());
SkAssertResult(fDataStream->rewind());
stream->writeText("\nendstream");
}
const char *
old_compress(const char *s)
{
static char buf[BUFFER_LEN];
char *to;
char token;
if (!old_table_initialized)
old_init_compress();
if (!s || compressed(s))
return s; /* already compressed */
/* tokenize the first characters */
for (to = buf; s[0] && s[1]; to++) {
if ((token = old_token_table[(int)s[0]][(int)s[1]])) {
*to = token;
s += 2;
} else {
*to = s[0];
s++;
}
}
/* copy the last character (if any) and null */
while ((*to++ = *s++)) ;
return buf;
}
const char *
old_uncompress(const char *s)
{
static char buf[BUFFER_LEN];
char *to;
const char *token;
if (!old_table_initialized)
old_init_compress();
if (!s || !compressed(s))
return s; /* already uncompressed */
for (to = buf; *s; s++) {
if (*s & TOKEN_BIT) {
token = old_tokens[*s & TOKEN_MASK];
*to++ = *token++;
*to++ = *token;
} else {
*to++ = *s;
}
}
*to++ = *s;
return buf;
}
template<class T> QVariant readBinaryArray(QDataStream& in, int& position) {
quint32 arrayLength;
quint32 encoding;
quint32 compressedLength;
in >> arrayLength;
in >> encoding;
in >> compressedLength;
position += sizeof(quint32) * 3;
QVector<T> values;
const unsigned int DEFLATE_ENCODING = 1;
if (encoding == DEFLATE_ENCODING) {
// preface encoded data with uncompressed length
QByteArray compressed(sizeof(quint32) + compressedLength, 0);
*((quint32*)compressed.data()) = qToBigEndian<quint32>(arrayLength * sizeof(T));
in.readRawData(compressed.data() + sizeof(quint32), compressedLength);
position += compressedLength;
QByteArray uncompressed = qUncompress(compressed);
QDataStream uncompressedIn(uncompressed);
uncompressedIn.setByteOrder(QDataStream::LittleEndian);
uncompressedIn.setVersion(QDataStream::Qt_4_5); // for single/double precision switch
for (quint32 i = 0; i < arrayLength; i++) {
T value;
uncompressedIn >> value;
values.append(value);
}
} else {
bool
MP4Reader::SkipVideoDemuxToNextKeyFrame(int64_t aTimeThreshold, uint32_t& parsed)
{
MOZ_ASSERT(GetTaskQueue()->IsCurrentThreadIn());
MOZ_ASSERT(mVideo.mDecoder);
Flush(TrackInfo::kVideoTrack);
// Loop until we reach the next keyframe after the threshold.
while (true) {
nsRefPtr<MediaRawData> compressed(PopSample(TrackInfo::kVideoTrack));
if (!compressed) {
// EOS, or error. This code assumes EOS, which may or may not be right.
MonitorAutoLock mon(mVideo.mMonitor);
mVideo.mDemuxEOS = true;
return false;
}
parsed++;
if (!compressed->mKeyframe ||
compressed->mTime < aTimeThreshold) {
continue;
}
mQueuedVideoSample = compressed;
break;
}
return true;
}
static void test_compressor_type(CompressorType compressor, CompressionLevel level, bool headerless, const Buffer &data, const set<CompressorType>::type &compressors)
{
OS_ASSERT(data.empty() == false);
shared_ptr<Buffer> compressed(OS_NEW Buffer());
BOOST_CHECK(CryptManager::instance()->compress(compressor, data.getData(), data.getSize(), *compressed, level));
Buffer decompressed;
BOOST_CHECK(CryptManager::instance()->decompress(compressor, compressed->getData(), compressed->getSize(), decompressed));
BOOST_CHECK(data == decompressed);
compressed->seekToBegin();
shared_ptr<MemFile> stream(OS_NEW MemFile(compressed));
if(headerless == false)
{
BOOST_CHECK(CryptManager::instance()->detectCompressor(compressed->getData(), compressed->getSize()) == compressor);
BOOST_CHECK(CryptManager::instance()->detectCompressor(stream) == compressor);
}
for(set<CompressorType>::type::const_iterator i = compressors.begin(); i != compressors.end(); ++i)
{
OS_ASSERT(*i != compressorTypeDeflate);
if(*i == compressor)
continue;
BOOST_CHECK(CryptManager::instance()->matchCompressor(compressed->getData(), compressed->getSize(), *i) == false);
BOOST_CHECK(CryptManager::instance()->matchCompressor(stream, *i) == false);
}
}
OSG_USING_NAMESPACE
int main (int argc, char **argv)
{
std::stringstream compressed(std::stringstream::in | std::stringstream::out);
// with false we add no gzip header and footer.
zip_ostream zout(compressed, false);
zout << std::string("Begin")
<< " " << Int32(123456789)
<< " " << Real32(1.41232)
<< " " << std::string("End");
// this flushes the stream and adds the gzip footer, this is
// done automatically in the zip_ostream destructor, but in this
// case we need to call it directly!
zout.finished();
std::cout << "Compressed : " << "'" << compressed.str() << "' (" << compressed.str().size() << ")" << std::endl;
std::string str1;
Int32 v1 = 0;
Real32 v2 = 0.0f;
std::string str2;
zip_istream zin(compressed);
zin >> str1 >> v1 >> v2 >> str2;
std::cout << "Uncompressed: " << "'" << str1 << "' " << v1 << " " << v2
<< " '" << str2 << "'" << std::endl;
return 0;
}
Data Gzip::Compress(const Data &data) {
CryptoPP::Gzip zipper;
zipper.Put((byte *) data.data(), data.size());
zipper.MessageEnd();
Data compressed(zipper.MaxRetrievable());
zipper.Get((byte *) compressed.data(), compressed.size());
return compressed;
}
uint64_t JoinPartition::writeByteStream(int which, ByteStream &bs)
{
size_t &offset = (which == 0 ? nextSmallOffset : nextLargeOffset);
fstream &fs = (which == 0 ? smallFile : largeFile);
const char *filename = (which == 0 ? smallFilename.c_str() : largeFilename.c_str());
fs.open(filename, ios::binary | ios::out | ios::app);
int saveErrno = errno;
if (!fs) {
fs.close();
ostringstream os;
os << "Disk join could not open file (write access) " << filename << ": " << strerror(saveErrno) << endl;
throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR);
}
uint64_t ret = 0;
size_t len = bs.length();
idbassert(len != 0);
fs.seekp(offset);
if (!useCompression) {
ret = len + 4;
fs.write((char *) &len, sizeof(len));
fs.write((char *) bs.buf(), len);
saveErrno = errno;
if (!fs) {
fs.close();
ostringstream os;
os << "Disk join could not write file " << filename << ": " << strerror(saveErrno) << endl;
throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR);
}
totalBytesWritten += sizeof(len) + len;
}
else {
uint64_t maxSize = compressor.maxCompressedSize(len);
size_t actualSize;
boost::scoped_array<uint8_t> compressed(new uint8_t[maxSize]);
compressor.compress((char *) bs.buf(), len, (char *) compressed.get(), &actualSize);
ret = actualSize + 4;
fs.write((char *) &actualSize, sizeof(actualSize));
fs.write((char *) compressed.get(), actualSize);
saveErrno = errno;
if (!fs) {
fs.close();
ostringstream os;
os << "Disk join could not write file " << filename << ": " << strerror(saveErrno) << endl;
throw IDBExcept(os.str().c_str(), ERR_DBJ_FILE_IO_ERROR);
}
totalBytesWritten += sizeof(actualSize) + actualSize;
}
bs.advance(len);
offset = fs.tellp();
fs.close();
return ret;
}
Array<uint8_t> compress(RawArray<const uint8_t> data, int level, event_t event) {
thread_time_t time(compress_kind,event);
if (level<20) { // zlib
size_t dest_size = compressBound(data.size());
Array<uint8_t> compressed(CHECK_CAST_INT(dest_size),uninit);
int z = compress2(compressed.data(),&dest_size,(uint8_t*)data.data(),data.size(),level);
if (z!=Z_OK)
THROW(IOError,"zlib failure in compress_and_write: %s",zlib_error(z));
return compressed.slice_own(0,CHECK_CAST_INT(dest_size));
} else { // lzma
size_t dest_size = lzma_stream_buffer_bound(data.size());
Array<uint8_t> compressed(CHECK_CAST_INT(dest_size),uninit);
size_t pos = 0;
lzma_ret r = lzma_easy_buffer_encode(level-20,LZMA_CHECK_CRC64,0,data.data(),data.size(),compressed.data(),&pos,dest_size);
if (r!=LZMA_OK)
THROW(RuntimeError,"lzma compression error: %s (%d)",lzma_error(r),r);
return compressed.slice_own(0,CHECK_CAST_INT(pos));
}
}
DEF_TEST(SkDeflateWStream, r) {
SkRandom random(123456);
for (int i = 0; i < 50; ++i) {
uint32_t size = random.nextULessThan(10000);
SkAutoTMalloc<uint8_t> buffer(size);
for (uint32_t j = 0; j < size; ++j) {
buffer[j] = random.nextU() & 0xff;
}
SkDynamicMemoryWStream dynamicMemoryWStream;
{
SkDeflateWStream deflateWStream(&dynamicMemoryWStream);
uint32_t j = 0;
while (j < size) {
uint32_t writeSize =
SkTMin(size - j, random.nextRangeU(1, 400));
if (!deflateWStream.write(&buffer[j], writeSize)) {
ERRORF(r, "something went wrong.");
return;
}
j += writeSize;
}
}
SkAutoTDelete<SkStreamAsset> compressed(
dynamicMemoryWStream.detachAsStream());
SkAutoTDelete<SkStreamAsset> decompressed(stream_inflate(compressed));
if (decompressed->getLength() != size) {
ERRORF(r, "Decompression failed to get right size [%d]."
" %u != %u", i, (unsigned)(decompressed->getLength()),
(unsigned)size);
SkString s = SkStringPrintf("/tmp/deftst_compressed_%d", i);
SkFILEWStream o(s.c_str());
o.writeStream(compressed.get(), compressed->getLength());
compressed->rewind();
s = SkStringPrintf("/tmp/deftst_input_%d", i);
SkFILEWStream o2(s.c_str());
o2.write(&buffer[0], size);
continue;
}
uint32_t minLength = SkTMin(size,
(uint32_t)(decompressed->getLength()));
for (uint32_t i = 0; i < minLength; ++i) {
uint8_t c;
SkDEBUGCODE(size_t rb =)decompressed->read(&c, sizeof(uint8_t));
SkASSERT(sizeof(uint8_t) == rb);
if (buffer[i] != c) {
ERRORF(r, "Decompression failed at byte %u.", (unsigned)i);
break;
}
}
}
}
long Engine<Dim,T,CompressibleBrickView>::
elementsCompressed() const
{
// If we are compressed, the number of compressed elements is the size
// of our domain; otherwise, it is zero.
if (compressed())
return domain().size();
else
return 0L;
}
void G2Node::SendQHT()
{
const Share::QueryHashTable& qht = System::GetShareMgr()->Qht();
const uint packetSize = 1024;
std::vector<char> inverted( qht.Size() );
uint count = 0;
for(uint i = 0; i < qht.Size(); ++i)
{
if(*(qht.Bytes() + i))
++count;
inverted.at(i) = 0xFF - *(qht.Bytes() + i);
}
//std::cout << count << "/" << qht.Size() << std::endl;
std::vector<char> compressed( ZLib::CompressBound( qht.Size() ) );
uint compressedSize;
try {
compressedSize = ZLib::Compress( &inverted[0], qht.Size(), &compressed[0], compressed.size() );
}
catch (std::exception& err)
{
System::LogBas() << "QHT compression error: " << err.what() << std::endl;
return;
}
//std::cout << "QHT SIZE: " << qht.Size() << " COMPRESSED: " << compressedSize << std::endl;
uint8 total = compressedSize / packetSize + (compressedSize % packetSize ? 1 : 0);
assert(compressedSize / packetSize + 1 < 256);
uint offset = 0;
for( uint8 i = 0; i < total; ++i)
{
G2::Packet pk;
G2::Writer w(pk);
w.Begin(G2::QHT);
w.Pod(char(1)); //command
w.Pod(char(i + 1)); //fragment number
w.Pod(char(total));
w.Pod(char(1)); //compressed
w.Pod(char(1)); //g2
const uint len = std::min(packetSize, compressedSize - offset);
w.Write( &compressed[offset], len );
offset += len;
w.Close(G2::QHT);
Send(pk);
}
}
int main()
{
_CrtSetDbgFlag(_CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF);
void* heap = HeapCreate(0, 0, 0);
MemoryBuffer compressed(heap);
MemoryBuffer plain(heap);
string st;
for (size_t index = 0; index < 1000; ++index)
{
st = st + "test";
}
ZipCompress(L"0", const_cast<char*>(st.c_str()), st.length() + 1, compressed);
ZipDecompress(L"0", compressed.GetData(), compressed.GetSize(), plain);
const char* output = reinterpret_cast<const char*>(plain.GetData());
/*
std::ifstream input("E:\\ticks.zip", std::ios::binary);
char buffer[256 * 1024] = "";
input.read(buffer, sizeof(buffer));
const size_t count = input.gcount();
HZIP handle = OpenZip(buffer, count, nullptr);
int index = -1;
ZIPENTRY zipEntry;
ZeroMemory(&zipEntry, sizeof(zipEntry));
ZRESULT status = FindZipItem(handle, L"ticks.txt", false, &index, &zipEntry);
char* uncompressed = new char[zipEntry.unc_size];
ZeroMemory(uncompressed, zipEntry.unc_size);
UnzipItem(handle, index, uncompressed, zipEntry.unc_size);
std::ofstream output("E:\\ticks2.txt", std::ios::binary);
output.write(uncompressed, zipEntry.unc_size);
CloseZip(handle);*/
return 0;
}
void chd_file::parse_v5_header(UINT8 *rawheader, sha1_t &parentsha1)
{
// verify header length
if (be_read(&rawheader[8], 4) != V5_HEADER_SIZE)
throw CHDERR_INVALID_FILE;
// extract core info
m_logicalbytes = be_read(&rawheader[32], 8);
m_mapoffset = be_read(&rawheader[40], 8);
m_metaoffset = be_read(&rawheader[48], 8);
m_hunkbytes = be_read(&rawheader[56], 4);
m_hunkcount = (m_logicalbytes + m_hunkbytes - 1) / m_hunkbytes;
m_unitbytes = be_read(&rawheader[60], 4);
m_unitcount = (m_logicalbytes + m_unitbytes - 1) / m_unitbytes;
// determine compression
m_compression[0] = be_read(&rawheader[16], 4);
m_compression[1] = be_read(&rawheader[20], 4);
m_compression[2] = be_read(&rawheader[24], 4);
m_compression[3] = be_read(&rawheader[28], 4);
m_allow_writes = !compressed();
// describe the format
m_mapoffset_offset = 40;
m_metaoffset_offset = 48;
m_sha1_offset = 84;
m_rawsha1_offset = 64;
m_parentsha1_offset = 104;
// determine properties of map entries
m_mapentrybytes = compressed() ? 12 : 4;
// extract parent SHA-1
parentsha1 = be_read_sha1(&rawheader[m_parentsha1_offset]);
}
bool Packet::uncompress()
{
unsigned char buf[ZT_PROTO_MAX_PACKET_LENGTH];
if ((compressed())&&(size() >= ZT_PROTO_MIN_PACKET_LENGTH)) {
if (size() > ZT_PACKET_IDX_PAYLOAD) {
unsigned int compLen = size() - ZT_PACKET_IDX_PAYLOAD;
int ucl = LZ4_decompress_safe((const char *)field(ZT_PACKET_IDX_PAYLOAD,compLen),(char *)buf,compLen,sizeof(buf));
if ((ucl > 0)&&(ucl <= (int)(capacity() - ZT_PACKET_IDX_PAYLOAD))) {
setSize((unsigned int)ucl + ZT_PACKET_IDX_PAYLOAD);
memcpy(field(ZT_PACKET_IDX_PAYLOAD,(unsigned int)ucl),buf,ucl);
} else return false;
}
(*this)[ZT_PACKET_IDX_VERB] &= (char)(~ZT_PROTO_VERB_FLAG_COMPRESSED);
}
return true;
}
bool Packet::compress()
{
unsigned char buf[ZT_PROTO_MAX_PACKET_LENGTH * 2];
if ((!compressed())&&(size() > (ZT_PACKET_IDX_PAYLOAD + 32))) {
int pl = (int)(size() - ZT_PACKET_IDX_PAYLOAD);
int cl = LZ4_compress((const char *)field(ZT_PACKET_IDX_PAYLOAD,(unsigned int)pl),(char *)buf,pl);
if ((cl > 0)&&(cl < pl)) {
(*this)[ZT_PACKET_IDX_VERB] |= (char)ZT_PROTO_VERB_FLAG_COMPRESSED;
setSize((unsigned int)cl + ZT_PACKET_IDX_PAYLOAD);
memcpy(field(ZT_PACKET_IDX_PAYLOAD,(unsigned int)cl),buf,cl);
return true;
}
}
(*this)[ZT_PACKET_IDX_VERB] &= (char)(~ZT_PROTO_VERB_FLAG_COMPRESSED);
return false;
}
// Compressed
void Index::getInvertedList(size_t pos, size_t size,
vector<term_info>& inverted_list) {
EliasGamma gamma;
inverted_list.clear();
size_t err = fseek(indexFile, pos, SEEK_SET);
if (err != 0) {
printf("Error finding inverted index location.");
exit(0);
}
// Read index file and uncompress
vector<unsigned char> compressed(size);
for (size_t i = 0; i < size; i++) {
unsigned int doc, freq;
term_info info;
fread(&doc, sizeof(unsigned int), 1, indexFile);
fread(&freq, sizeof(unsigned int), 1, indexFile);
info.doc = doc;
info.freq = freq;
inverted_list.push_back(info);
}
vector<unsigned int> serial_buffer;
gamma.decode(compressed, serial_buffer);
term_info info;
for (size_t i = 0; i < serial_buffer.size(); i += 2) {
if (i == 0) {
info.doc = serial_buffer[i];
} else {
info.doc = inverted_list[inverted_list.size() - 1].doc
+ serial_buffer[i];
}
info.freq = serial_buffer[i + 1];
inverted_list.push_back(info);
}
}
static bool AppendEntry(str::Str<char>& data, str::Str<char>& content, const WCHAR *filePath, const char *inArchiveName, lzma::FileInfo *fi=NULL)
{
size_t nameLen = str::Len(inArchiveName);
CrashIf(nameLen > UINT32_MAX - 25);
uint32_t headerSize = 25 + (uint32_t)nameLen;
FILETIME ft = file::GetModificationTime(filePath);
if (fi && FileTimeEq(ft, fi->ftModified)) {
ReusePrevious:
ByteWriterLE meta(data.AppendBlanks(24), 24);
meta.Write32(headerSize);
meta.Write32(fi->compressedSize);
meta.Write32(fi->uncompressedSize);
meta.Write32(fi->uncompressedCrc32);
meta.Write32(ft.dwLowDateTime);
meta.Write32(ft.dwHighDateTime);
data.Append(inArchiveName, nameLen + 1);
return content.AppendChecked(fi->compressedData, fi->compressedSize);
}
size_t fileDataLen;
ScopedMem<char> fileData(file::ReadAll(filePath, &fileDataLen));
if (!fileData || fileDataLen >= UINT32_MAX) {
fprintf(stderr, "Failed to read \"%S\" for compression\n", filePath);
return false;
}
uint32_t fileDataCrc = crc32(0, (const uint8_t *)fileData.Get(), (uint32_t)fileDataLen);
if (fi && fi->uncompressedCrc32 == fileDataCrc && fi->uncompressedSize == fileDataLen)
goto ReusePrevious;
size_t compressedSize = fileDataLen + 1;
ScopedMem<char> compressed((char *)malloc(compressedSize));
if (!compressed)
return false;
if (!Compress(fileData, fileDataLen, compressed, &compressedSize))
return false;
ByteWriterLE meta(data.AppendBlanks(24), 24);
meta.Write32(headerSize);
meta.Write32((uint32_t)compressedSize);
meta.Write32((uint32_t)fileDataLen);
meta.Write32(fileDataCrc);
meta.Write32(ft.dwLowDateTime);
meta.Write32(ft.dwHighDateTime);
data.Append(inArchiveName, nameLen + 1);
return content.AppendChecked(compressed, compressedSize);
}
GTEST_TEST(DEFLATE, decompressStream) {
static const size_t kSizeCompressed = sizeof(kDataCompressed);
static const size_t kSizeDecompressed = strlen(kDataUncompressed);
Common::MemoryReadStream compressed(kDataCompressed);
Common::SeekableReadStream *decompressed =
Common::decompressDeflate(compressed, kSizeCompressed, kSizeDecompressed, Common::kWindowBitsMaxRaw);
ASSERT_NE(decompressed, static_cast<Common::SeekableReadStream *>(0));
ASSERT_EQ(decompressed->size(), kSizeDecompressed);
for (size_t i = 0; i < kSizeDecompressed; i++)
EXPECT_EQ(decompressed->readByte(), kDataUncompressed[i]) << "At index " << i;
delete decompressed;
}
static void write_compressed_image(FILE* f, Image* image)
{
PixelIO<ImageTraits> pixel_io;
z_stream zstream;
int y, err;
zstream.zalloc = (alloc_func)0;
zstream.zfree = (free_func)0;
zstream.opaque = (voidpf)0;
err = deflateInit(&zstream, Z_DEFAULT_COMPRESSION);
if (err != Z_OK)
throw base::Exception("ZLib error %d in deflateInit().", err);
std::vector<uint8_t> scanline(ImageTraits::scanline_size(image->w));
std::vector<uint8_t> compressed(4096);
for (y=0; y<image->h; y++) {
typename ImageTraits::address_t address = image_address_fast<ImageTraits>(image, 0, y);
pixel_io.write_scanline(address, image->w, &scanline[0]);
zstream.next_in = (Bytef*)&scanline[0];
zstream.avail_in = scanline.size();
do {
zstream.next_out = (Bytef*)&compressed[0];
zstream.avail_out = compressed.size();
// Compress
err = deflate(&zstream, (y < image->h-1 ? Z_NO_FLUSH: Z_FINISH));
if (err != Z_OK && err != Z_STREAM_END)
throw base::Exception("ZLib error %d in deflate().", err);
int output_bytes = compressed.size() - zstream.avail_out;
if (output_bytes > 0) {
if ((fwrite(&compressed[0], 1, output_bytes, f) != (size_t)output_bytes)
|| ferror(f))
throw base::Exception("Error writing compressed image pixels.\n");
}
} while (zstream.avail_out == 0);
}
err = deflateEnd(&zstream);
if (err != Z_OK)
throw base::Exception("ZLib error %d in deflateEnd().", err);
}
int LuaUtils::ZlibCompress(lua_State* L)
{
size_t inLen;
const char* inData = luaL_checklstring(L, 1, &inLen);
long unsigned bufsize = inLen*1.02+32;
std::vector<boost::uint8_t> compressed(bufsize, 0);
const int error = compress(&compressed[0], &bufsize, (const boost::uint8_t*)inData, inLen);
if (error == Z_OK)
{
lua_pushlstring(L, (const char*)&compressed[0], bufsize);
return 1;
}
else
{
return luaL_error(L, "Error while compressing");
}
}
void StraightRodPair::buildFull(const RodTemplate& rodTemplate) {
double startZ = startZMode() == StartZMode::MODULECENTER ? -(*rodTemplate.begin())->length()/2. : 0.;
auto zListPair = computeZListPair(rodTemplate.begin(), rodTemplate.end(), startZ, 0);
// actual module creation
// CUIDADO log rod balancing effort
buildModules(zPlusModules_, rodTemplate, zListPair.first, BuildDir::RIGHT, zPlusParity(), 1);
double currMaxZ = zPlusModules_.size() > 1 ? MAX(zPlusModules_.rbegin()->planarMaxZ(), (zPlusModules_.rbegin()+1)->planarMaxZ()) : (!zPlusModules_.empty() ? zPlusModules_.rbegin()->planarMaxZ() : 0.);
// CUIDADO this only checks the positive side... the negative side might actually have a higher fabs(maxZ) if the barrel is long enough and there's an inversion
buildModules(zMinusModules_, rodTemplate, zListPair.second, BuildDir::LEFT, -zPlusParity(), -1);
auto collisionsZPlus = solveCollisionsZPlus();
auto collisionsZMinus = solveCollisionsZMinus();
if (!collisionsZPlus.empty() || !collisionsZMinus.empty()) logWARNING("Some modules have been translated to avoid collisions. Check info tab");
if (compressed() && maxZ.state() && currMaxZ > maxZ()) compressToZ(maxZ());
currMaxZ = zPlusModules_.size() > 1 ? MAX(zPlusModules_.rbegin()->planarMaxZ(), (zPlusModules_.rbegin()+1)->planarMaxZ()) : (!zPlusModules_.empty() ? zPlusModules_.rbegin()->planarMaxZ() : 0.);
maxZ(currMaxZ);
}
bool compressor::load(FILE* f, binary_decoder& data) {
uint32_t uncompressed_len, compressed_len, poor_crc;
unsigned char props_encoded[LZMA_PROPS_SIZE];
if (fread(&uncompressed_len, sizeof(uncompressed_len), 1, f) != 1) return false;
if (fread(&compressed_len, sizeof(compressed_len), 1, f) != 1) return false;
if (fread(&poor_crc, sizeof(poor_crc), 1, f) != 1) return false;
if (poor_crc != uncompressed_len * 19991 + compressed_len * 199999991 + 1234567890) return false;
if (fread(props_encoded, sizeof(props_encoded), 1, f) != 1) return false;
vector<unsigned char> compressed(compressed_len);
if (fread(compressed.data(), 1, compressed_len, f) != compressed_len) return false;
lzma::ELzmaStatus status;
size_t uncompressed_size = uncompressed_len, compressed_size = compressed_len;
auto res = lzma::LzmaDecode(data.fill(uncompressed_len), &uncompressed_size, compressed.data(), &compressed_size, props_encoded, LZMA_PROPS_SIZE, lzma::LZMA_FINISH_ANY, &status, &lzmaAllocator);
if (res != SZ_OK || uncompressed_size != uncompressed_len || compressed_size != compressed_len) return false;
return true;
}
void chd_file::create_open_common()
{
// verify the compression types and initialize the codecs
for (int decompnum = 0; decompnum < ARRAY_LENGTH(m_compression); decompnum++)
{
m_decompressor[decompnum] = chd_codec_list::new_decompressor(m_compression[decompnum], *this);
if (m_decompressor[decompnum] == NULL && m_compression[decompnum] != 0)
throw CHDERR_UNKNOWN_COMPRESSION;
}
// read the map; v5+ compressed drives need to read and decompress their map
m_rawmap.resize(m_hunkcount * m_mapentrybytes);
if (m_version >= 5 && compressed())
decompress_v5_map();
else
file_read(m_mapoffset, m_rawmap, m_rawmap.count());
// allocate the temporary compressed buffer and a buffer for caching
m_compressed.resize(m_hunkbytes);
m_cache.resize(m_hunkbytes);
}
bool compressor::save(FILE* f, const binary_encoder& enc) {
size_t uncompressed_size = enc.data.size(), compressed_size = 2 * enc.data.size() + 100;
vector<unsigned char> compressed(compressed_size);
lzma::CLzmaEncProps props;
lzma::LzmaEncProps_Init(&props);
unsigned char props_encoded[LZMA_PROPS_SIZE];
size_t props_encoded_size = LZMA_PROPS_SIZE;
auto res = lzma::LzmaEncode(compressed.data(), &compressed_size, enc.data.data(), uncompressed_size, &props, props_encoded, &props_encoded_size, 0, nullptr, &lzmaAllocator, &lzmaAllocator);
if (res != SZ_OK) return false;
uint32_t poor_crc = uncompressed_size * 19991 + compressed_size * 199999991 + 1234567890;
if (uint32_t(uncompressed_size) != uncompressed_size || uint32_t(compressed_size) != compressed_size) return false;
if (fwrite(&uncompressed_size, sizeof(uint32_t), 1, f) != 1) return false;
if (fwrite(&compressed_size, sizeof(uint32_t), 1, f) != 1) return false;
if (fwrite(&poor_crc, sizeof(uint32_t), 1, f) != 1) return false;
if (fwrite(props_encoded, sizeof(props_encoded), 1, f) != 1) return false;
if (fwrite(compressed.data(), 1, compressed_size, f) != compressed_size) return false;
return true;
}
