unsigned int XXH32 (const void* input, size_t len, unsigned seed)
{
#if 0
// Simple version, good for code maintenance, but unfortunately slow for small inputs
XXH32_state_t state;
XXH32_reset(&state, seed);
XXH32_update(&state, input, len);
return XXH32_digest(&state);
#else
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
# if !defined(XXH_USE_UNALIGNED_ACCESS)
if ((((size_t)input) & 3) == 0) // Input is aligned, let's leverage the speed advantage
{
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
else
return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
}
# endif
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
else
return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
#endif
}
static uint32_t src_hash(void *_k) {
srckey *k = (srckey *)_k;
XXH32_state_t h;
/* seed is an arbitrary, but fixed value */
XXH32_reset(&h, 42);
XXH32_update(&h, (void *)k->src, k->len);
XXH32_update(&h, (void *)k->arch, sizeof(k->arch));
return XXH32_digest(&h);
}
/*
* Class: net_jpountz_xxhash_XXHashJNI
* Method: XXH32_init
* Signature: (I)J
*/
JNIEXPORT jlong JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH32_1init
(JNIEnv *env, jclass cls, jint seed) {
XXH32_state_t *state = XXH32_createState();
if (XXH32_reset(state, seed) != XXH_OK) {
XXH32_freeState(state);
throw_OOM(env);
return 0;
}
return (jlong) state;
}
void xxHashStream::Reset(dword seed)
{
XXH32_reset((XXH32_state_t *)context, seed);
}
/*
Compressed format : MAGICNUMBER - STREAMDESCRIPTOR - ( BLOCKHEADER - COMPRESSEDBLOCK ) - STREAMCRC
MAGICNUMBER - 4 bytes - Designates compression algo
STREAMDESCRIPTOR - 1 byte
bits 0-3 : max block size, 2^value from 0 to 0xA; min 0=>1KB, max 0x6=>64KB, typical 5=>32 KB
bits 4-7 = 0 : reserved;
BLOCKHEADER - 1-5 bytes
1st byte :
bits 6-7 : blockType (compressed, raw, rle, crc (end of Frame)
bit 5 : full block
** if not full block **
2nd & 3rd byte : regenerated size of block (big endian); note : 0 = 64 KB
** if blockType==compressed **
next 2 bytes : compressed size of block
COMPRESSEDBLOCK
the compressed data itself.
STREAMCRC - 3 bytes (including 1-byte blockheader)
22 bits (xxh32() >> 5) checksum of the original data, big endian
*/
unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename)
{
U64 filesize = 0;
U64 compressedfilesize = 0;
char* in_buff;
char* out_buff;
FILE* finput;
FILE* foutput;
size_t sizeCheck;
size_t inputBlockSize = FIO_blockID_to_blockSize(g_blockSizeId);
XXH32_state_t xxhState;
typedef size_t (*compressor_t) (void* dst, size_t dstSize, const void* src, size_t srcSize);
compressor_t compressor;
unsigned magicNumber;
/* Init */
XXH32_reset (&xxhState, FSE_CHECKSUM_SEED);
get_fileHandle(input_filename, output_filename, &finput, &foutput);
switch (g_compressor)
{
case FIO_fse:
compressor = FSE_compress;
magicNumber = FIO_magicNumber_fse;
break;
case FIO_huff0:
compressor = HUF_compress;
magicNumber = FIO_magicNumber_huff0;
break;
case FIO_zlibh:
compressor = FIO_ZLIBH_compress;
magicNumber = FIO_magicNumber_zlibh;
break;
default :
EXM_THROW(20, "unknown compressor selection");
}
/* Allocate Memory */
if (inputBlockSize==0) EXM_THROW(0, "impossible problem, to please static analyzer");
in_buff = (char*)malloc(inputBlockSize);
out_buff = (char*)malloc(FSE_compressBound(inputBlockSize) + 5);
if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory");
/* Write Frame Header */
FIO_writeLE32(out_buff, magicNumber);
out_buff[4] = (char)g_blockSizeId; /* Max Block Size descriptor */
sizeCheck = fwrite(out_buff, 1, FIO_FRAMEHEADERSIZE, foutput);
if (sizeCheck!=FIO_FRAMEHEADERSIZE) EXM_THROW(22, "Write error : cannot write header");
compressedfilesize += FIO_FRAMEHEADERSIZE;
/* Main compression loop */
while (1)
{
/* Fill input Buffer */
size_t cSize;
size_t inSize = fread(in_buff, (size_t)1, (size_t)inputBlockSize, finput);
if (inSize==0) break;
filesize += inSize;
XXH32_update(&xxhState, in_buff, inSize);
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20));
/* Compress Block */
cSize = compressor(out_buff + FIO_maxBlockHeaderSize, FSE_compressBound(inputBlockSize), in_buff, inSize);
if (FSE_isError(cSize)) EXM_THROW(23, "Compression error : %s ", FSE_getErrorName(cSize));
/* Write cBlock */
switch(cSize)
{
size_t headerSize;
case 0: /* raw */
if (inSize == inputBlockSize)
{
out_buff[0] = (BYTE)((bt_raw << 6) + BIT5);
headerSize = 1;
}
else
{
out_buff[2] = (BYTE)inSize;
out_buff[1] = (BYTE)(inSize >> 8);
out_buff[0] = (BYTE)(bt_raw << 6);
headerSize = 3;
}
sizeCheck = fwrite(out_buff, 1, headerSize, foutput);
if (sizeCheck!=headerSize) EXM_THROW(24, "Write error : cannot write block header");
sizeCheck = fwrite(in_buff, 1, inSize, foutput);
if (sizeCheck!=(size_t)(inSize)) EXM_THROW(25, "Write error : cannot write block");
compressedfilesize += inSize + headerSize;
break;
case 1: /* rle */
if (inSize == inputBlockSize)
{
out_buff[0] = (BYTE)((bt_rle << 6) + BIT5);
headerSize = 1;
}
else
{
out_buff[2] = (BYTE)inSize;
out_buff[1] = (BYTE)(inSize >> 8);
out_buff[0] = (BYTE)(bt_raw << 6);
headerSize = 3;
}
out_buff[headerSize] = in_buff[0];
sizeCheck = fwrite(out_buff, 1, headerSize+1, foutput);
if (sizeCheck!=(headerSize+1)) EXM_THROW(26, "Write error : cannot write rle block");
compressedfilesize += headerSize + 1;
break;
default : /* compressed */
if (inSize == inputBlockSize)
{
out_buff[2] = (BYTE)((bt_compressed << 6) + BIT5);
out_buff[3] = (BYTE)(cSize >> 8);
out_buff[4] = (BYTE)cSize;
headerSize = 3;
}
else
{
out_buff[0] = (BYTE)(bt_compressed << 6);
out_buff[1] = (BYTE)(inSize >> 8);
out_buff[2] = (BYTE)inSize;
out_buff[3] = (BYTE)(cSize >> 8);
out_buff[4] = (BYTE)cSize;
headerSize = FIO_maxBlockHeaderSize;
}
sizeCheck = fwrite(out_buff+(FIO_maxBlockHeaderSize-headerSize), 1, headerSize+cSize, foutput);
if (sizeCheck!=(headerSize+cSize)) EXM_THROW(27, "Write error : cannot write rle block");
compressedfilesize += headerSize + cSize;
break;
}
