EXTERN_C_ENTER
JNIEXPORT jint JNICALL JavaCritical_org_lwjgl_util_lz4_LZ4HC_nLZ4_1compress_1HC(jlong srcAddress, jlong dstAddress, jint srcSize, jint dstCapacity, jint compressionLevel) {
char const *src = (char const *)(intptr_t)srcAddress;
char *dst = (char *)(intptr_t)dstAddress;
return (jint)LZ4_compress_HC(src, dst, srcSize, dstCapacity, compressionLevel);
}
static VALUE rubylz4_raw_compressHC(VALUE self, VALUE input, VALUE compression_level_value)
{
Check_Type(input, RUBY_T_STRING);
const char* input_data = RSTRING_PTR(input);
int input_size = RSTRING_LEN(input);
Check_Type(compression_level_value, RUBY_T_FIXNUM);
int compression_level = FIX2INT(compression_level_value);
// do compress
int max_compressed_size = LZ4_compressBound(input_size);
VALUE output = rb_str_new(NULL, max_compressed_size);
char* output_data = RSTRING_PTR(output);
int compressed_size = LZ4_compress_HC(input_data, output_data,
input_size, max_compressed_size,
compression_level);
if (compressed_size == 0) {
rb_raise(rb_eRuntimeError, "%s", "compress failed");
} else {
rb_str_resize(output, compressed_size);
return output;
}
}
EXTERN_C_ENTER
JNIEXPORT jint JNICALL Java_org_lwjgl_util_lz4_LZ4HC_nLZ4_1compress_1HC(JNIEnv *__env, jclass clazz, jlong srcAddress, jlong dstAddress, jint srcSize, jint dstCapacity, jint compressionLevel) {
char const *src = (char const *)(intptr_t)srcAddress;
char *dst = (char *)(intptr_t)dstAddress;
UNUSED_PARAMS(__env, clazz)
return (jint)LZ4_compress_HC(src, dst, srcSize, dstCapacity, compressionLevel);
}
int lz4compress_size( const unsigned char* data, unsigned size ){
auto max_size = LZ4_compressBound( size );
std::vector<unsigned char> buffer( max_size );
return LZ4_compress_HC(
(const char*)data, (char*)buffer.data()
, size, buffer.size()
, LZ4HC_CLEVEL_MAX //TODO: Higher?
);
}
std::pair<std::unique_ptr<char[]>, size_t> compress(const void* data, size_t dataSize, int level)
{
if (dataSize == 0) return std::make_pair(std::unique_ptr<char[]>(), 0);
int csizeBound = LZ4_compressBound(dataSize);
std::unique_ptr<char[]> cdata(new char[csizeBound]);
int csize = (level == 0)
? LZ4_compress_default(static_cast<const char*>(data), cdata.get(), dataSize, csizeBound)
: LZ4_compress_HC(static_cast<const char*>(data), cdata.get(), dataSize, csizeBound, level);
assert(csize >= 0 && csize <= csizeBound);
return std::make_pair(std::move(cdata), csize);
}
static PyObject *compress_hc(PyObject *self, PyObject *args, PyObject *keywds)
{
(void)self;
const char *source;
int source_size;
int compression_level;
static char *kwlist[] = {"source", "compression_level", NULL};
if (!PyArg_ParseTupleAndKeywords(args, keywds, "s#i", kwlist,
&source, &source_size,
&compression_level)) {
return NULL;
}
if (source_size > LZ4_MAX_INPUT_SIZE) {
PyErr_Format(PyExc_ValueError, "input data is %d bytes, which is larger than the maximum supported size of %d bytes", source_size, LZ4_MAX_INPUT_SIZE);
return NULL;
}
int dest_size = LZ4_compressBound(source_size);
PyObject *py_dest = PyBytes_FromStringAndSize(NULL, dest_size);
if (py_dest == NULL) {
return NULL;
}
char *dest = PyBytes_AS_STRING(py_dest);
if (source_size > 0) {
int compressed_size = LZ4_compress_HC(source, dest, source_size, dest_size, compression_level);
if (compressed_size <= 0) {
Py_DECREF(py_dest);
PyErr_Format(PyExc_RuntimeError, "LZ4_compress_HC failed with code: %d", compressed_size);
return NULL;
}
/* The actual compressed size might be less than we allocated
(we allocated using a worst case guess). If the actual size is
less than 75% of what we allocated, then it's worth performing an
expensive resize operation to reclaim some space. */
if (compressed_size < (dest_size / 4) * 3) {
_PyBytes_Resize(&py_dest, compressed_size);
} else {
Py_SIZE(py_dest) = compressed_size;
}
}
return py_dest;
}
int64_t lzbench_lz4hc_compress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t level, size_t, char*)
{
return LZ4_compress_HC(inbuf, outbuf, insize, outsize, level);
}
static ZEND_FUNCTION(lz4_compress)
{
zval *data;
char *output;
int output_len, data_len, dst_len;
long level = 0;
long maxLevel = (long)PHP_LZ4_CLEVEL_MAX;
char *extra = NULL;
#if ZEND_MODULE_API_NO >= 20141001
size_t extra_len = -1;
#else
int extra_len = -1;
#endif
int offset = 0;
if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC,
"z|ls", &data, &level,
&extra, &extra_len) == FAILURE) {
RETURN_FALSE;
}
if (Z_TYPE_P(data) != IS_STRING) {
zend_error(E_WARNING,
"lz4_compress : expects parameter to be string.");
RETURN_FALSE;
}
if (extra && extra_len > 0) {
offset = extra_len;
} else {
offset = sizeof(int);
}
data_len = Z_STRLEN_P(data);
dst_len = LZ4_compressBound(data_len) + offset;
output = (char *)emalloc(dst_len);
if (!output) {
zend_error(E_WARNING, "lz4_compress : memory error");
RETURN_FALSE;
}
if (extra && extra_len > 0) {
memcpy(output, extra, offset);
} else {
/* Set the data length */
memcpy(output, &data_len, offset);
}
if (level == 0) {
output_len = LZ4_compress_default(Z_STRVAL_P(data), output + offset, data_len, dst_len - offset - 1);
} else {
if (level > maxLevel || level < 0) {
zend_error(E_WARNING, "lz4_compress: compression level (%ld)"
" must be within 1..%ld", level, maxLevel);
efree(output);
RETURN_FALSE;
}
output_len = LZ4_compress_HC(Z_STRVAL_P(data), output + offset, data_len, dst_len - offset - 1, level);
}
if (output_len <= 0) {
RETVAL_FALSE;
} else {
#if ZEND_MODULE_API_NO >= 20141001
RETVAL_STRINGL(output, output_len + offset);
#else
RETVAL_STRINGL(output, output_len + offset, 1);
#endif
}
efree(output);
}
void CompressedWriteBuffer::nextImpl()
{
if (!offset())
return;
size_t uncompressed_size = offset();
size_t compressed_size = 0;
char * compressed_buffer_ptr = nullptr;
/** The format of compressed block - see CompressedStream.h
*/
switch (method)
{
case CompressionMethod::LZ4:
case CompressionMethod::LZ4HC:
{
static constexpr size_t header_size = 1 + sizeof(UInt32) + sizeof(UInt32);
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wold-style-cast"
compressed_buffer.resize(header_size + LZ4_COMPRESSBOUND(uncompressed_size));
#pragma GCC diagnostic pop
compressed_buffer[0] = static_cast<UInt8>(CompressionMethodByte::LZ4);
if (method == CompressionMethod::LZ4)
compressed_size = header_size + LZ4_compress_default(
working_buffer.begin(),
&compressed_buffer[header_size],
uncompressed_size,
LZ4_COMPRESSBOUND(uncompressed_size));
else
compressed_size = header_size + LZ4_compress_HC(
working_buffer.begin(),
&compressed_buffer[header_size],
uncompressed_size,
LZ4_COMPRESSBOUND(uncompressed_size),
0);
UInt32 compressed_size_32 = compressed_size;
UInt32 uncompressed_size_32 = uncompressed_size;
unalignedStore(&compressed_buffer[1], compressed_size_32);
unalignedStore(&compressed_buffer[5], uncompressed_size_32);
compressed_buffer_ptr = &compressed_buffer[0];
break;
}
case CompressionMethod::ZSTD:
{
static constexpr size_t header_size = 1 + sizeof(UInt32) + sizeof(UInt32);
compressed_buffer.resize(header_size + ZSTD_compressBound(uncompressed_size));
compressed_buffer[0] = static_cast<UInt8>(CompressionMethodByte::ZSTD);
size_t res = ZSTD_compress(
&compressed_buffer[header_size],
compressed_buffer.size(),
working_buffer.begin(),
uncompressed_size,
1);
if (ZSTD_isError(res))
throw Exception("Cannot compress block with ZSTD: " + std::string(ZSTD_getErrorName(res)), ErrorCodes::CANNOT_COMPRESS);
compressed_size = header_size + res;
UInt32 compressed_size_32 = compressed_size;
UInt32 uncompressed_size_32 = uncompressed_size;
unalignedStore(&compressed_buffer[1], compressed_size_32);
unalignedStore(&compressed_buffer[5], uncompressed_size_32);
compressed_buffer_ptr = &compressed_buffer[0];
break;
}
default:
throw Exception("Unknown compression method", ErrorCodes::UNKNOWN_COMPRESSION_METHOD);
}
CityHash_v1_0_2::uint128 checksum = CityHash_v1_0_2::CityHash128(compressed_buffer_ptr, compressed_size);
out.write(reinterpret_cast<const char *>(&checksum), sizeof(checksum));
out.write(compressed_buffer_ptr, compressed_size);
}
void BuildAddFile(FilePackerBuilder& builder, bool compress, const std::string& base, const std::string& path, const char* name)
{
FileEntryBuilder entrybuilder;
FileEntry& entry = entrybuilder.entry;
entry.path = path + name;
std::string full_path = base + "/" + entry.path;
FILE* file = fopen(full_path.c_str(), "rb");
if (file != NULL)
{
fseek(file, 0, SEEK_END);
entry.header.uncompr_size = ftell(file);
fseek(file, 0, SEEK_SET);
unsigned char* buffer = new unsigned char[entry.header.uncompr_size];
fread(buffer, entry.header.uncompr_size, 1, file);
fclose(file);
unsigned char* crc_buffer = NULL;
if (compress)
{
unsigned char* lz4_out_bound = NULL;
int lz4_size = 0;
int lz4_size_bound = LZ4_compressBound(entry.header.uncompr_size);
lz4_out_bound = new unsigned char[lz4_size_bound];
lz4_size = LZ4_compress_HC((const char*)buffer, (char*)lz4_out_bound, entry.header.uncompr_size, lz4_size_bound, 9);
uint32_t ratio = (entry.header.uncompr_size / 2) + (entry.header.uncompr_size / 4); // < 75% original size is ok
if (lz4_size < ratio)
{
entry.header.compression = FileEntry::Header::LZ4HC;
entry.header.size = lz4_size;
entrybuilder.compressed_data = new unsigned char[entry.header.size];
memcpy(entrybuilder.compressed_data, lz4_out_bound, entry.header.size);
crc_buffer = entrybuilder.compressed_data;
}
delete[] lz4_out_bound;
}
if (crc_buffer == NULL)
{
entry.header.compression = FileEntry::Header::UNCOMPRESSED;
entry.header.size = entry.header.uncompr_size;
entrybuilder.compressed_data = NULL;
crc_buffer = buffer;
}
crcFast crc;
for (uint32_t i = 0; i < entry.header.size; i++)
crc.Append(crc_buffer[i]);
entry.header.crc = crc.CRC();
entry.header.offset = builder.current_offset;
builder.current_offset += entry.header.size;
delete[] buffer;
entry.header.crc = crc.CRC();
builder.entries.push_back(entrybuilder);
PrintFileEntry(entry);
}
}
inline void images_to_gv(std::string output_path, std::vector<std::string> imagePaths, float fps, int *done_frames, bool hasAlpha, std::shared_ptr<Dx11> dx, ofxCoroutine::Yield &yield) {
if (imagePaths.empty()) {
return;
}
// memory
uint32_t _width = 0;
uint32_t _height = 0;
float _fps = fps;
uint32_t _bufferSize = 0;
std::vector<uint8_t> _lz4CompressBuffer;
std::vector<Lz4Block> _lz4blocks;
std::unique_ptr<GpuVideoIO> _io;
int _index = 0;
int width;
int height;
ofPixels img;
ofLoadImage(img, imagePaths[0]);
width = img.getWidth();
height = img.getHeight();
_width = width;
_height = height;
int blockcount = ((_width + 3) / 4) * ((_height + 3) / 4);
int blocksize = 16;
_bufferSize = blockcount * blocksize;
// 書き出し開始
_io = std::unique_ptr<GpuVideoIO>(new GpuVideoIO(output_path.c_str(), "wb"));
// ヘッダー情報書き出し
#define W(v) if(_io->write(&v, sizeof(v)) != sizeof(v)) { assert(0); }
W(_width);
W(_height);
uint32_t frameCount = (uint32_t)imagePaths.size();
W(frameCount);
W(_fps);
uint32_t videoFmt = GPU_COMPRESS_BC7;
W(videoFmt);
W(_bufferSize);
#undef W
int compressBound = LZ4_compressBound(_bufferSize);
_lz4CompressBuffer.resize(compressBound);
for (int i = 0; i < imagePaths.size(); ++i) {
DirectX::TexMetadata metadata;
DirectX::ScratchImage image;
auto imgPath = ofToDataPath(imagePaths[i]);
HRESULT hr = DirectX::LoadFromWICFile(to_wstring(imgPath).c_str(), 0, &metadata, image);
if (FAILED(hr)) {
abort();
}
DWORD flags = DirectX::TEX_COMPRESS_DEFAULT;
flags |= DirectX::TEX_COMPRESS_PARALLEL;
flags |= DirectX::TEX_COMPRESS_BC7_USE_3SUBSETS;
flags |= DirectX::TEX_COMPRESS_UNIFORM;
flags |= DirectX::TEX_COMPRESS_SRGB_IN;
flags |= DirectX::TEX_COMPRESS_SRGB_OUT;
float alphaWeight = hasAlpha ? 1.0f : 0.0f;
DirectX::ScratchImage cImage;
hr = DirectX::Compress(dx11->device(), *image.GetImage(0, 0, 0), DXGI_FORMAT_BC7_UNORM_SRGB, flags, alphaWeight, cImage);
int src = cImage.GetPixelsSize();
if (_bufferSize != cImage.GetPixelsSize()) {
abort();
}
int compressed = LZ4_compress_HC((char *)cImage.GetPixels(),
(char *)_lz4CompressBuffer.data(),
_bufferSize, compressBound, LZ4HC_CLEVEL_MAX);
// 住所を記録しつつ
uint64_t head = _lz4blocks.empty() ? kRawMemoryAt : (_lz4blocks[_lz4blocks.size() - 1].address + _lz4blocks[_lz4blocks.size() - 1].size);
Lz4Block lz4block;
lz4block.address = head;
lz4block.size = compressed;
_lz4blocks.push_back(lz4block);
// 書き込み
if (_io->write(_lz4CompressBuffer.data(), compressed) != compressed) {
assert(0);
}
(*done_frames)++;
yield();
}
// 最後に住所を記録
uint64_t size = _lz4blocks.size() * sizeof(Lz4Block);
if (_io->write(_lz4blocks.data(), size) != size) {
abort();
}
// ファイルをクローズ
_io.reset();
//for (;;) {
// if (_index < imagePaths.size()) {
// auto compress = [imagePaths, _width, _height, _squishFlag](int index, uint8_t *dst) {
// std::string src = imagePaths[index];
// ofPixels img;
// ofLoadImage(img, src);
// img.setImageType(OF_IMAGE_COLOR_ALPHA);
// squish::CompressImage(img.getData(), _width, _height, dst, _squishFlag);
// };
// const int kBatchCount = 32;
// int workCount = std::min((int)imagePaths.size() - _index, kBatchCount);
// uint32_t lz4sizes[kBatchCount];
// int compressBound = LZ4_compressBound(_bufferSize);
// _gpuCompressBuffer.resize(workCount * _bufferSize);
// _lz4CompressBuffer.resize(workCount * compressBound);
// tbb::parallel_for(tbb::blocked_range<int>(0, workCount, 1), [compress, _index, _bufferSize, compressBound, &lz4sizes, &_gpuCompressBuffer, &_lz4CompressBuffer, &done_frames](const tbb::blocked_range< int >& range) {
// for (int i = range.begin(); i != range.end(); i++) {
// compress(_index + i, _gpuCompressBuffer.data() + i * _bufferSize);
// lz4sizes[i] = LZ4_compress_HC((char *)_gpuCompressBuffer.data() + i * _bufferSize,
// (char *)_lz4CompressBuffer.data() + i * compressBound,
// _bufferSize, compressBound, 16);
// done_frames++;
// }
// });
// uint64_t head = _lz4blocks.empty() ? kRawMemoryAt : (_lz4blocks[_lz4blocks.size() - 1].address + _lz4blocks[_lz4blocks.size() - 1].size);
// for (int i = 0; i < workCount; i++) {
// // 住所を記録しつつ
// Lz4Block lz4block;
// lz4block.address = head;
// lz4block.size = lz4sizes[i];
// head += lz4block.size;
// _lz4blocks.push_back(lz4block);
// // 書き込み
// if (_io->write(_lz4CompressBuffer.data() + i * compressBound, lz4sizes[i]) != lz4sizes[i]) {
// assert(0);
// }
// }
// _index += workCount;
// // 強制離脱
// if (interrupt) {
// _io.reset();
// ::remove(output_path.c_str());
// break;
// }
// }
// else {
// // 最後に住所を記録
// uint64_t size = _lz4blocks.size() * sizeof(Lz4Block);
// if (_io->write(_lz4blocks.data(), size) != size) {
// assert(0);
// }
// // ファイルをクローズ
// _io.reset();
// // 終了
// break;
// }
//}
}
static int dump_song_kr4(char *filename1, char *filename2,
int debug, int size,
char *outfile, char *a, char *t, int force_end_frame) {
int result;
int data_size;
int x,y;
struct ym_song_t ym_song,ym_song2;
int num_chunks;
FILE *fff;
char outname[BUFSIZ];
int j;
int minutes,seconds;
int compressed_size;
int fake_frames;
unsigned char *interleaved_data;
char *raw_data,*compressed_data;
unsigned char frame[YM5_FRAME_SIZE];
int skip;
unsigned char temp_value;
char title[BUFSIZ];
char author[BUFSIZ];
int end_frame;
/* FIXME: if "-" then use stdout? */
sprintf(outname,"%s",outfile);
fff=fopen(outname,"w");
if (fff==NULL) {
fprintf(stderr,"Error opening %s\n",outname);
return -1;
}
fprintf(stderr, "\nDumping song %s+%s to %s\n",
filename1,filename2,outname);
/*****************************************/
/* LOAD SONG */
/***************************************/
result=load_ym_song(filename1,&ym_song);
if (result<0) {
return -1;
}
result=load_ym_song(filename2,&ym_song2);
if (result<0) {
return -1;
}
if (force_end_frame) {
end_frame=force_end_frame;
}
else {
end_frame=ym_song.num_frames;
}
seconds=end_frame/ym_song.frame_rate;
minutes=seconds/60;
seconds-=(minutes*60);
if (minutes>9) {
fprintf(stderr,"Warning! Decoder doesn't "
"necessarily handle files > 9min\n");
}
fprintf(stderr,"\tFrames: %d, %d:%02d\n",
end_frame,minutes,seconds);
fputc('K',fff);
fputc('R',fff);
fputc('W',fff);
/* default */
// fprintf(fff,"INTRO2: JUNGAR OF BIT WORLD FROM KIEV%c",0);
// fprintf(fff,"BY: SURGEON (ALEKSEY LUTSENKO)%c",0);
/* Get from title */
if (t) {
strcpy(title,t);
}
else {
strcpy(title,ym_song.song_name);
}
title[40]=0;
if (a) {
sprintf(author,"%s",a);
}
else {
sprintf(author,"BY: %s",ym_song.author);
}
author[40]=0;
fprintf(stderr,"TITLE: %s\n",title);
fprintf(stderr,"AUTHOR: %s\n",author);
skip=1+3+3+
strlen(title)+
strlen(author)+
strlen("0:00 / 0:00");
fputc(skip,fff);
fputc( (40-strlen(title))/2,fff);
fprintf(fff,title);
fputc(0,fff);
fputc( (40-strlen(author))/2,fff);
fprintf(fff,author);
fputc(0,fff);
fputc(13,fff);
fprintf(fff,"0:00 / %2d:%02d",minutes,seconds);
fputc(0,fff);
/******************/
/* Play the song! */
/******************/
/* plus one for end frame */
// 8960 /768 = 11.6
// num_chunks = 12
// 8960/50=179.2 = 2:59
// stops at 2:48 = 8400 (11 is 8448)
num_chunks=(end_frame+1)/(pages_per_chunk*256);
/* pad to even number of frames */
num_chunks+=1;
data_size=num_chunks*pages_per_chunk*256*14;
fake_frames=data_size/14;
fprintf(stderr,"%d frames %d fake_frames %d chunks total total_size %d\n",
end_frame,fake_frames,num_chunks,data_size);
interleaved_data=calloc(data_size,sizeof(char));
if (interleaved_data==NULL) {
fprintf(stderr,"Error allocating memory!\n");
return -1;
}
/* 0xFFs at end are end-of-song marker */
/* Tricky if exact multiple of 256 :( */
memset(interleaved_data,0xff,data_size);
raw_data=calloc(pages_per_chunk*256*14,sizeof(char));
if (raw_data==NULL) {
fprintf(stderr,"Error allocating memory!\n");
return -1;
}
compressed_data=calloc(pages_per_chunk*256*14*2,sizeof(char));
if (raw_data==NULL) {
fprintf(stderr,"Error allocating memory!\n");
return -1;
}
/* Interleave the data */
for(y=0;y<14;y++) {
for(x=0;x<end_frame;x++) {
if (y<11) {
ym_return_frame(&ym_song,x,frame,NULL);
interleaved_data[(y*fake_frames)+x]=
frame[y];
}
else {
ym_return_frame(&ym_song2,x,frame,NULL);
if (y==11) temp_value=frame[2];
if (y==12) temp_value=frame[3];
if (y==13) temp_value=frame[9];
interleaved_data[(y*fake_frames)+x]=
temp_value;
}
}
}
/* HACK! make sure we have end-marker */
interleaved_data[(1*fake_frames)+(end_frame-1)]=0xff;
for(j=0;j<num_chunks;j++) {
for(y=0;y<14;y++) {
for(x=0;x<(256*pages_per_chunk);x++) {
raw_data[x+y*(256*pages_per_chunk)]=
interleaved_data[x+
j*(256*pages_per_chunk)+
(y*fake_frames)];
}
}
// printf("%d: ",j);
// for(y=0;y<14;y++) {
// printf("%02X ",
// (unsigned char)raw_data[(255*pages_per_chunk)+(y*256*pages_per_chunk)]);
// }
// printf("\n");
compressed_size=LZ4_compress_HC (raw_data,
compressed_data,
256*pages_per_chunk*14,
256*pages_per_chunk*14*2,
16);
if (compressed_size>65536) {
fprintf(stderr,"Error! Compressed data too big!\n");
}
fputc(compressed_size%256,fff);
fputc(compressed_size/256,fff);
fwrite(compressed_data,sizeof(unsigned char),
compressed_size,fff);
// fprintf(stderr,"%d size %x\n",j,compressed_size);
// for(y=0;y<14;y++) {
// for(x=0;x<(256*pages_per_chunk);x++) {
// fputc(raw_data[x+y*(256*pages_per_chunk)],fff);
// }
// }
}
fputc(0,fff);
fputc(0,fff);
fclose(fff);
fprintf(stderr,"; Total size = %d bytes\n",end_frame*14);
free(interleaved_data);
free(raw_data);
free(compressed_data);
/* Free the ym file */
free(ym_song.file_data);
return 0;
}
