EXTERN_C_ENTER JNIEXPORT jint JNICALL JavaCritical_org_lwjgl_util_lz4_LZ4HC_nLZ4_1compress_1HC(jlong srcAddress, jlong dstAddress, jint srcSize, jint dstCapacity, jint compressionLevel) { char const *src = (char const *)(intptr_t)srcAddress; char *dst = (char *)(intptr_t)dstAddress; return (jint)LZ4_compress_HC(src, dst, srcSize, dstCapacity, compressionLevel); }
static VALUE rubylz4_raw_compressHC(VALUE self, VALUE input, VALUE compression_level_value) { Check_Type(input, RUBY_T_STRING); const char* input_data = RSTRING_PTR(input); int input_size = RSTRING_LEN(input); Check_Type(compression_level_value, RUBY_T_FIXNUM); int compression_level = FIX2INT(compression_level_value); // do compress int max_compressed_size = LZ4_compressBound(input_size); VALUE output = rb_str_new(NULL, max_compressed_size); char* output_data = RSTRING_PTR(output); int compressed_size = LZ4_compress_HC(input_data, output_data, input_size, max_compressed_size, compression_level); if (compressed_size == 0) { rb_raise(rb_eRuntimeError, "%s", "compress failed"); } else { rb_str_resize(output, compressed_size); return output; } }
EXTERN_C_ENTER JNIEXPORT jint JNICALL Java_org_lwjgl_util_lz4_LZ4HC_nLZ4_1compress_1HC(JNIEnv *__env, jclass clazz, jlong srcAddress, jlong dstAddress, jint srcSize, jint dstCapacity, jint compressionLevel) { char const *src = (char const *)(intptr_t)srcAddress; char *dst = (char *)(intptr_t)dstAddress; UNUSED_PARAMS(__env, clazz) return (jint)LZ4_compress_HC(src, dst, srcSize, dstCapacity, compressionLevel); }
int lz4compress_size( const unsigned char* data, unsigned size ){ auto max_size = LZ4_compressBound( size ); std::vector<unsigned char> buffer( max_size ); return LZ4_compress_HC( (const char*)data, (char*)buffer.data() , size, buffer.size() , LZ4HC_CLEVEL_MAX //TODO: Higher? ); }
std::pair<std::unique_ptr<char[]>, size_t> compress(const void* data, size_t dataSize, int level) { if (dataSize == 0) return std::make_pair(std::unique_ptr<char[]>(), 0); int csizeBound = LZ4_compressBound(dataSize); std::unique_ptr<char[]> cdata(new char[csizeBound]); int csize = (level == 0) ? LZ4_compress_default(static_cast<const char*>(data), cdata.get(), dataSize, csizeBound) : LZ4_compress_HC(static_cast<const char*>(data), cdata.get(), dataSize, csizeBound, level); assert(csize >= 0 && csize <= csizeBound); return std::make_pair(std::move(cdata), csize); }
static PyObject *compress_hc(PyObject *self, PyObject *args, PyObject *keywds) { (void)self; const char *source; int source_size; int compression_level; static char *kwlist[] = {"source", "compression_level", NULL}; if (!PyArg_ParseTupleAndKeywords(args, keywds, "s#i", kwlist, &source, &source_size, &compression_level)) { return NULL; } if (source_size > LZ4_MAX_INPUT_SIZE) { PyErr_Format(PyExc_ValueError, "input data is %d bytes, which is larger than the maximum supported size of %d bytes", source_size, LZ4_MAX_INPUT_SIZE); return NULL; } int dest_size = LZ4_compressBound(source_size); PyObject *py_dest = PyBytes_FromStringAndSize(NULL, dest_size); if (py_dest == NULL) { return NULL; } char *dest = PyBytes_AS_STRING(py_dest); if (source_size > 0) { int compressed_size = LZ4_compress_HC(source, dest, source_size, dest_size, compression_level); if (compressed_size <= 0) { Py_DECREF(py_dest); PyErr_Format(PyExc_RuntimeError, "LZ4_compress_HC failed with code: %d", compressed_size); return NULL; } /* The actual compressed size might be less than we allocated (we allocated using a worst case guess). If the actual size is less than 75% of what we allocated, then it's worth performing an expensive resize operation to reclaim some space. */ if (compressed_size < (dest_size / 4) * 3) { _PyBytes_Resize(&py_dest, compressed_size); } else { Py_SIZE(py_dest) = compressed_size; } } return py_dest; }
int64_t lzbench_lz4hc_compress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t level, size_t, char*) { return LZ4_compress_HC(inbuf, outbuf, insize, outsize, level); }
static ZEND_FUNCTION(lz4_compress) { zval *data; char *output; int output_len, data_len, dst_len; long level = 0; long maxLevel = (long)PHP_LZ4_CLEVEL_MAX; char *extra = NULL; #if ZEND_MODULE_API_NO >= 20141001 size_t extra_len = -1; #else int extra_len = -1; #endif int offset = 0; if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "z|ls", &data, &level, &extra, &extra_len) == FAILURE) { RETURN_FALSE; } if (Z_TYPE_P(data) != IS_STRING) { zend_error(E_WARNING, "lz4_compress : expects parameter to be string."); RETURN_FALSE; } if (extra && extra_len > 0) { offset = extra_len; } else { offset = sizeof(int); } data_len = Z_STRLEN_P(data); dst_len = LZ4_compressBound(data_len) + offset; output = (char *)emalloc(dst_len); if (!output) { zend_error(E_WARNING, "lz4_compress : memory error"); RETURN_FALSE; } if (extra && extra_len > 0) { memcpy(output, extra, offset); } else { /* Set the data length */ memcpy(output, &data_len, offset); } if (level == 0) { output_len = LZ4_compress_default(Z_STRVAL_P(data), output + offset, data_len, dst_len - offset - 1); } else { if (level > maxLevel || level < 0) { zend_error(E_WARNING, "lz4_compress: compression level (%ld)" " must be within 1..%ld", level, maxLevel); efree(output); RETURN_FALSE; } output_len = LZ4_compress_HC(Z_STRVAL_P(data), output + offset, data_len, dst_len - offset - 1, level); } if (output_len <= 0) { RETVAL_FALSE; } else { #if ZEND_MODULE_API_NO >= 20141001 RETVAL_STRINGL(output, output_len + offset); #else RETVAL_STRINGL(output, output_len + offset, 1); #endif } efree(output); }
void CompressedWriteBuffer::nextImpl() { if (!offset()) return; size_t uncompressed_size = offset(); size_t compressed_size = 0; char * compressed_buffer_ptr = nullptr; /** The format of compressed block - see CompressedStream.h */ switch (method) { case CompressionMethod::LZ4: case CompressionMethod::LZ4HC: { static constexpr size_t header_size = 1 + sizeof(UInt32) + sizeof(UInt32); #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wold-style-cast" compressed_buffer.resize(header_size + LZ4_COMPRESSBOUND(uncompressed_size)); #pragma GCC diagnostic pop compressed_buffer[0] = static_cast<UInt8>(CompressionMethodByte::LZ4); if (method == CompressionMethod::LZ4) compressed_size = header_size + LZ4_compress_default( working_buffer.begin(), &compressed_buffer[header_size], uncompressed_size, LZ4_COMPRESSBOUND(uncompressed_size)); else compressed_size = header_size + LZ4_compress_HC( working_buffer.begin(), &compressed_buffer[header_size], uncompressed_size, LZ4_COMPRESSBOUND(uncompressed_size), 0); UInt32 compressed_size_32 = compressed_size; UInt32 uncompressed_size_32 = uncompressed_size; unalignedStore(&compressed_buffer[1], compressed_size_32); unalignedStore(&compressed_buffer[5], uncompressed_size_32); compressed_buffer_ptr = &compressed_buffer[0]; break; } case CompressionMethod::ZSTD: { static constexpr size_t header_size = 1 + sizeof(UInt32) + sizeof(UInt32); compressed_buffer.resize(header_size + ZSTD_compressBound(uncompressed_size)); compressed_buffer[0] = static_cast<UInt8>(CompressionMethodByte::ZSTD); size_t res = ZSTD_compress( &compressed_buffer[header_size], compressed_buffer.size(), working_buffer.begin(), uncompressed_size, 1); if (ZSTD_isError(res)) throw Exception("Cannot compress block with ZSTD: " + std::string(ZSTD_getErrorName(res)), ErrorCodes::CANNOT_COMPRESS); compressed_size = header_size + res; UInt32 compressed_size_32 = compressed_size; UInt32 uncompressed_size_32 = uncompressed_size; unalignedStore(&compressed_buffer[1], compressed_size_32); unalignedStore(&compressed_buffer[5], uncompressed_size_32); compressed_buffer_ptr = &compressed_buffer[0]; break; } default: throw Exception("Unknown compression method", ErrorCodes::UNKNOWN_COMPRESSION_METHOD); } CityHash_v1_0_2::uint128 checksum = CityHash_v1_0_2::CityHash128(compressed_buffer_ptr, compressed_size); out.write(reinterpret_cast<const char *>(&checksum), sizeof(checksum)); out.write(compressed_buffer_ptr, compressed_size); }
void BuildAddFile(FilePackerBuilder& builder, bool compress, const std::string& base, const std::string& path, const char* name) { FileEntryBuilder entrybuilder; FileEntry& entry = entrybuilder.entry; entry.path = path + name; std::string full_path = base + "/" + entry.path; FILE* file = fopen(full_path.c_str(), "rb"); if (file != NULL) { fseek(file, 0, SEEK_END); entry.header.uncompr_size = ftell(file); fseek(file, 0, SEEK_SET); unsigned char* buffer = new unsigned char[entry.header.uncompr_size]; fread(buffer, entry.header.uncompr_size, 1, file); fclose(file); unsigned char* crc_buffer = NULL; if (compress) { unsigned char* lz4_out_bound = NULL; int lz4_size = 0; int lz4_size_bound = LZ4_compressBound(entry.header.uncompr_size); lz4_out_bound = new unsigned char[lz4_size_bound]; lz4_size = LZ4_compress_HC((const char*)buffer, (char*)lz4_out_bound, entry.header.uncompr_size, lz4_size_bound, 9); uint32_t ratio = (entry.header.uncompr_size / 2) + (entry.header.uncompr_size / 4); // < 75% original size is ok if (lz4_size < ratio) { entry.header.compression = FileEntry::Header::LZ4HC; entry.header.size = lz4_size; entrybuilder.compressed_data = new unsigned char[entry.header.size]; memcpy(entrybuilder.compressed_data, lz4_out_bound, entry.header.size); crc_buffer = entrybuilder.compressed_data; } delete[] lz4_out_bound; } if (crc_buffer == NULL) { entry.header.compression = FileEntry::Header::UNCOMPRESSED; entry.header.size = entry.header.uncompr_size; entrybuilder.compressed_data = NULL; crc_buffer = buffer; } crcFast crc; for (uint32_t i = 0; i < entry.header.size; i++) crc.Append(crc_buffer[i]); entry.header.crc = crc.CRC(); entry.header.offset = builder.current_offset; builder.current_offset += entry.header.size; delete[] buffer; entry.header.crc = crc.CRC(); builder.entries.push_back(entrybuilder); PrintFileEntry(entry); } }
inline void images_to_gv(std::string output_path, std::vector<std::string> imagePaths, float fps, int *done_frames, bool hasAlpha, std::shared_ptr<Dx11> dx, ofxCoroutine::Yield &yield) { if (imagePaths.empty()) { return; } // memory uint32_t _width = 0; uint32_t _height = 0; float _fps = fps; uint32_t _bufferSize = 0; std::vector<uint8_t> _lz4CompressBuffer; std::vector<Lz4Block> _lz4blocks; std::unique_ptr<GpuVideoIO> _io; int _index = 0; int width; int height; ofPixels img; ofLoadImage(img, imagePaths[0]); width = img.getWidth(); height = img.getHeight(); _width = width; _height = height; int blockcount = ((_width + 3) / 4) * ((_height + 3) / 4); int blocksize = 16; _bufferSize = blockcount * blocksize; // 書き出し開始 _io = std::unique_ptr<GpuVideoIO>(new GpuVideoIO(output_path.c_str(), "wb")); // ヘッダー情報書き出し #define W(v) if(_io->write(&v, sizeof(v)) != sizeof(v)) { assert(0); } W(_width); W(_height); uint32_t frameCount = (uint32_t)imagePaths.size(); W(frameCount); W(_fps); uint32_t videoFmt = GPU_COMPRESS_BC7; W(videoFmt); W(_bufferSize); #undef W int compressBound = LZ4_compressBound(_bufferSize); _lz4CompressBuffer.resize(compressBound); for (int i = 0; i < imagePaths.size(); ++i) { DirectX::TexMetadata metadata; DirectX::ScratchImage image; auto imgPath = ofToDataPath(imagePaths[i]); HRESULT hr = DirectX::LoadFromWICFile(to_wstring(imgPath).c_str(), 0, &metadata, image); if (FAILED(hr)) { abort(); } DWORD flags = DirectX::TEX_COMPRESS_DEFAULT; flags |= DirectX::TEX_COMPRESS_PARALLEL; flags |= DirectX::TEX_COMPRESS_BC7_USE_3SUBSETS; flags |= DirectX::TEX_COMPRESS_UNIFORM; flags |= DirectX::TEX_COMPRESS_SRGB_IN; flags |= DirectX::TEX_COMPRESS_SRGB_OUT; float alphaWeight = hasAlpha ? 1.0f : 0.0f; DirectX::ScratchImage cImage; hr = DirectX::Compress(dx11->device(), *image.GetImage(0, 0, 0), DXGI_FORMAT_BC7_UNORM_SRGB, flags, alphaWeight, cImage); int src = cImage.GetPixelsSize(); if (_bufferSize != cImage.GetPixelsSize()) { abort(); } int compressed = LZ4_compress_HC((char *)cImage.GetPixels(), (char *)_lz4CompressBuffer.data(), _bufferSize, compressBound, LZ4HC_CLEVEL_MAX); // 住所を記録しつつ uint64_t head = _lz4blocks.empty() ? kRawMemoryAt : (_lz4blocks[_lz4blocks.size() - 1].address + _lz4blocks[_lz4blocks.size() - 1].size); Lz4Block lz4block; lz4block.address = head; lz4block.size = compressed; _lz4blocks.push_back(lz4block); // 書き込み if (_io->write(_lz4CompressBuffer.data(), compressed) != compressed) { assert(0); } (*done_frames)++; yield(); } // 最後に住所を記録 uint64_t size = _lz4blocks.size() * sizeof(Lz4Block); if (_io->write(_lz4blocks.data(), size) != size) { abort(); } // ファイルをクローズ _io.reset(); //for (;;) { // if (_index < imagePaths.size()) { // auto compress = [imagePaths, _width, _height, _squishFlag](int index, uint8_t *dst) { // std::string src = imagePaths[index]; // ofPixels img; // ofLoadImage(img, src); // img.setImageType(OF_IMAGE_COLOR_ALPHA); // squish::CompressImage(img.getData(), _width, _height, dst, _squishFlag); // }; // const int kBatchCount = 32; // int workCount = std::min((int)imagePaths.size() - _index, kBatchCount); // uint32_t lz4sizes[kBatchCount]; // int compressBound = LZ4_compressBound(_bufferSize); // _gpuCompressBuffer.resize(workCount * _bufferSize); // _lz4CompressBuffer.resize(workCount * compressBound); // tbb::parallel_for(tbb::blocked_range<int>(0, workCount, 1), [compress, _index, _bufferSize, compressBound, &lz4sizes, &_gpuCompressBuffer, &_lz4CompressBuffer, &done_frames](const tbb::blocked_range< int >& range) { // for (int i = range.begin(); i != range.end(); i++) { // compress(_index + i, _gpuCompressBuffer.data() + i * _bufferSize); // lz4sizes[i] = LZ4_compress_HC((char *)_gpuCompressBuffer.data() + i * _bufferSize, // (char *)_lz4CompressBuffer.data() + i * compressBound, // _bufferSize, compressBound, 16); // done_frames++; // } // }); // uint64_t head = _lz4blocks.empty() ? kRawMemoryAt : (_lz4blocks[_lz4blocks.size() - 1].address + _lz4blocks[_lz4blocks.size() - 1].size); // for (int i = 0; i < workCount; i++) { // // 住所を記録しつつ // Lz4Block lz4block; // lz4block.address = head; // lz4block.size = lz4sizes[i]; // head += lz4block.size; // _lz4blocks.push_back(lz4block); // // 書き込み // if (_io->write(_lz4CompressBuffer.data() + i * compressBound, lz4sizes[i]) != lz4sizes[i]) { // assert(0); // } // } // _index += workCount; // // 強制離脱 // if (interrupt) { // _io.reset(); // ::remove(output_path.c_str()); // break; // } // } // else { // // 最後に住所を記録 // uint64_t size = _lz4blocks.size() * sizeof(Lz4Block); // if (_io->write(_lz4blocks.data(), size) != size) { // assert(0); // } // // ファイルをクローズ // _io.reset(); // // 終了 // break; // } //} }
static int dump_song_kr4(char *filename1, char *filename2, int debug, int size, char *outfile, char *a, char *t, int force_end_frame) { int result; int data_size; int x,y; struct ym_song_t ym_song,ym_song2; int num_chunks; FILE *fff; char outname[BUFSIZ]; int j; int minutes,seconds; int compressed_size; int fake_frames; unsigned char *interleaved_data; char *raw_data,*compressed_data; unsigned char frame[YM5_FRAME_SIZE]; int skip; unsigned char temp_value; char title[BUFSIZ]; char author[BUFSIZ]; int end_frame; /* FIXME: if "-" then use stdout? */ sprintf(outname,"%s",outfile); fff=fopen(outname,"w"); if (fff==NULL) { fprintf(stderr,"Error opening %s\n",outname); return -1; } fprintf(stderr, "\nDumping song %s+%s to %s\n", filename1,filename2,outname); /*****************************************/ /* LOAD SONG */ /***************************************/ result=load_ym_song(filename1,&ym_song); if (result<0) { return -1; } result=load_ym_song(filename2,&ym_song2); if (result<0) { return -1; } if (force_end_frame) { end_frame=force_end_frame; } else { end_frame=ym_song.num_frames; } seconds=end_frame/ym_song.frame_rate; minutes=seconds/60; seconds-=(minutes*60); if (minutes>9) { fprintf(stderr,"Warning! Decoder doesn't " "necessarily handle files > 9min\n"); } fprintf(stderr,"\tFrames: %d, %d:%02d\n", end_frame,minutes,seconds); fputc('K',fff); fputc('R',fff); fputc('W',fff); /* default */ // fprintf(fff,"INTRO2: JUNGAR OF BIT WORLD FROM KIEV%c",0); // fprintf(fff,"BY: SURGEON (ALEKSEY LUTSENKO)%c",0); /* Get from title */ if (t) { strcpy(title,t); } else { strcpy(title,ym_song.song_name); } title[40]=0; if (a) { sprintf(author,"%s",a); } else { sprintf(author,"BY: %s",ym_song.author); } author[40]=0; fprintf(stderr,"TITLE: %s\n",title); fprintf(stderr,"AUTHOR: %s\n",author); skip=1+3+3+ strlen(title)+ strlen(author)+ strlen("0:00 / 0:00"); fputc(skip,fff); fputc( (40-strlen(title))/2,fff); fprintf(fff,title); fputc(0,fff); fputc( (40-strlen(author))/2,fff); fprintf(fff,author); fputc(0,fff); fputc(13,fff); fprintf(fff,"0:00 / %2d:%02d",minutes,seconds); fputc(0,fff); /******************/ /* Play the song! */ /******************/ /* plus one for end frame */ // 8960 /768 = 11.6 // num_chunks = 12 // 8960/50=179.2 = 2:59 // stops at 2:48 = 8400 (11 is 8448) num_chunks=(end_frame+1)/(pages_per_chunk*256); /* pad to even number of frames */ num_chunks+=1; data_size=num_chunks*pages_per_chunk*256*14; fake_frames=data_size/14; fprintf(stderr,"%d frames %d fake_frames %d chunks total total_size %d\n", end_frame,fake_frames,num_chunks,data_size); interleaved_data=calloc(data_size,sizeof(char)); if (interleaved_data==NULL) { fprintf(stderr,"Error allocating memory!\n"); return -1; } /* 0xFFs at end are end-of-song marker */ /* Tricky if exact multiple of 256 :( */ memset(interleaved_data,0xff,data_size); raw_data=calloc(pages_per_chunk*256*14,sizeof(char)); if (raw_data==NULL) { fprintf(stderr,"Error allocating memory!\n"); return -1; } compressed_data=calloc(pages_per_chunk*256*14*2,sizeof(char)); if (raw_data==NULL) { fprintf(stderr,"Error allocating memory!\n"); return -1; } /* Interleave the data */ for(y=0;y<14;y++) { for(x=0;x<end_frame;x++) { if (y<11) { ym_return_frame(&ym_song,x,frame,NULL); interleaved_data[(y*fake_frames)+x]= frame[y]; } else { ym_return_frame(&ym_song2,x,frame,NULL); if (y==11) temp_value=frame[2]; if (y==12) temp_value=frame[3]; if (y==13) temp_value=frame[9]; interleaved_data[(y*fake_frames)+x]= temp_value; } } } /* HACK! make sure we have end-marker */ interleaved_data[(1*fake_frames)+(end_frame-1)]=0xff; for(j=0;j<num_chunks;j++) { for(y=0;y<14;y++) { for(x=0;x<(256*pages_per_chunk);x++) { raw_data[x+y*(256*pages_per_chunk)]= interleaved_data[x+ j*(256*pages_per_chunk)+ (y*fake_frames)]; } } // printf("%d: ",j); // for(y=0;y<14;y++) { // printf("%02X ", // (unsigned char)raw_data[(255*pages_per_chunk)+(y*256*pages_per_chunk)]); // } // printf("\n"); compressed_size=LZ4_compress_HC (raw_data, compressed_data, 256*pages_per_chunk*14, 256*pages_per_chunk*14*2, 16); if (compressed_size>65536) { fprintf(stderr,"Error! Compressed data too big!\n"); } fputc(compressed_size%256,fff); fputc(compressed_size/256,fff); fwrite(compressed_data,sizeof(unsigned char), compressed_size,fff); // fprintf(stderr,"%d size %x\n",j,compressed_size); // for(y=0;y<14;y++) { // for(x=0;x<(256*pages_per_chunk);x++) { // fputc(raw_data[x+y*(256*pages_per_chunk)],fff); // } // } } fputc(0,fff); fputc(0,fff); fclose(fff); fprintf(stderr,"; Total size = %d bytes\n",end_frame*14); free(interleaved_data); free(raw_data); free(compressed_data); /* Free the ym file */ free(ym_song.file_data); return 0; }