static uint32_t src_hash(void *_k) {
srckey *k = (srckey *)_k;
XXH32_state_t h;
/* seed is an arbitrary, but fixed value */
XXH32_reset(&h, 42);
XXH32_update(&h, (void *)k->src, k->len);
XXH32_update(&h, (void *)k->arch, sizeof(k->arch));
return XXH32_digest(&h);
}
unsigned int XXH32 (const void* input, size_t len, unsigned seed)
{
#if 0
// Simple version, good for code maintenance, but unfortunately slow for small inputs
XXH32_state_t state;
XXH32_reset(&state, seed);
XXH32_update(&state, input, len);
return XXH32_digest(&state);
#else
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
# if !defined(XXH_USE_UNALIGNED_ACCESS)
if ((((size_t)input) & 3) == 0) // Input is aligned, let's leverage the speed advantage
{
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
else
return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
}
# endif
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
else
return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
#endif
}
bool Xxh32::update(const void* input, int len) {
Lock lock(mut);
if(st) {
return XXH_OK == XXH32_update(st.get(), input, len);
} else {
return false;
}
}
Xxh32::Xxh32(const void* input, int len, uint32_t seed)
: mut()
, st(new char[XXH32_sizeofState()])
{
Lock lock(mut);
XXH32_resetState(st.get(), seed);
XXH32_update(st.get(), input, len);
}
static void fill_xxhash(struct verify_header *hdr, void *p, unsigned int len)
{
struct vhdr_xxhash *vh = hdr_priv(hdr);
void *state;
state = XXH32_init(1);
XXH32_update(state, p, len);
vh->hash = XXH32_digest(state);
}
static PyObject* _update_hash(void* hash_state, PyObject* arg_obj) {
Py_ssize_t tuple_length;
Py_ssize_t tuple_i;
PyObject* tuple_obj, *partial_result;
#if PY_MAJOR_VERSION >= 3
if (PyBytes_Check(arg_obj)) {
XXH32_update(hash_state, PyBytes_AsString(arg_obj), PyBytes_Size(arg_obj));
}
#else
if (PyString_Check(arg_obj)) {
XXH32_update(hash_state, PyString_AsString(arg_obj), PyString_Size(arg_obj));
}
#endif
else if (PyByteArray_Check(arg_obj)) {
XXH32_update(hash_state, PyByteArray_AsString(arg_obj), PyByteArray_Size(arg_obj));
}
else if (PyTuple_Check(arg_obj)) {
tuple_length = PyTuple_GET_SIZE(arg_obj);
for(tuple_i = 0; tuple_i < tuple_length; tuple_i++) {
tuple_obj = PyTuple_GetItem(arg_obj, tuple_i);
partial_result = _update_hash(hash_state, tuple_obj);
// Check exceptions
if (partial_result == NULL) return NULL;
}
}
else if (arg_obj == Py_None) {
Py_RETURN_NONE;
}
else if (PyUnicode_Check(arg_obj)) {
PyErr_SetString(PyExc_TypeError, "Found unicode string, you must convert to bytes/str before hashing.");
return NULL;
}
else {
PyErr_Format(PyExc_TypeError, "Tried to hash unsupported type: %S.", Py_TYPE(arg_obj));
return NULL;
}
Py_RETURN_NONE;
}
/*
* Class: net_jpountz_xxhash_XXHashJNI
* Method: XXH32_update
* Signature: (J[BII)V
*/
JNIEXPORT void JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH32_1update
(JNIEnv *env, jclass cls, jlong state, jbyteArray src, jint off, jint len) {
char* in = (char*) (*env)->GetPrimitiveArrayCritical(env, src, 0);
if (in == NULL) {
throw_OOM(env);
return;
}
XXH32_update((void*) state, in + off, len);
(*env)->ReleasePrimitiveArrayCritical(env, src, in, 0);
}
static uint64_t t_xxhash(void)
{
void *state;
struct timeval s;
uint64_t ret;
void *buf;
int i;
state = XXH32_init(0x8989);
buf = malloc(CHUNK);
randomize_buf(buf, CHUNK, 0x8989);
fio_gettime(&s, NULL);
for (i = 0; i < NR_CHUNKS; i++)
XXH32_update(state, buf, CHUNK);
XXH32_digest(state);
ret = utime_since_now(&s);
free(buf);
return ret;
}
static int verify_io_u_xxhash(struct verify_header *hdr, struct vcont *vc)
{
void *p = io_u_verify_off(hdr, vc);
struct vhdr_xxhash *vh = hdr_priv(hdr);
uint32_t hash;
void *state;
dprint(FD_VERIFY, "xxhash verify io_u %p, len %u\n", vc->io_u, hdr->len);
state = XXH32_init(1);
XXH32_update(state, p, hdr->len - hdr_size(vc->td, hdr));
hash = XXH32_digest(state);
if (vh->hash == hash)
return 0;
vc->name = "xxhash";
vc->good_crc = &vh->hash;
vc->bad_crc = &hash;
vc->crc_len = sizeof(hash);
log_verify_failure(hdr, vc);
return EILSEQ;
}
void xxHashStream::Out(const void *data, dword size)
{
XXH32_update((XXH32_state_t *)context, data, size);
}
/*
Compressed format : MAGICNUMBER - STREAMDESCRIPTOR - ( BLOCKHEADER - COMPRESSEDBLOCK ) - STREAMCRC
MAGICNUMBER - 4 bytes - Designates compression algo
STREAMDESCRIPTOR - 1 byte
bits 0-3 : max block size, 2^value from 0 to 0xA; min 0=>1KB, max 0x6=>64KB, typical 5=>32 KB
bits 4-7 = 0 : reserved;
BLOCKHEADER - 1-5 bytes
1st byte :
bits 6-7 : blockType (compressed, raw, rle, crc (end of Frame)
bit 5 : full block
** if not full block **
2nd & 3rd byte : regenerated size of block (big endian); note : 0 = 64 KB
** if blockType==compressed **
next 2 bytes : compressed size of block
COMPRESSEDBLOCK
the compressed data itself.
STREAMCRC - 3 bytes (including 1-byte blockheader)
22 bits (xxh32() >> 5) checksum of the original data, big endian
*/
unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename)
{
U64 filesize = 0;
U64 compressedfilesize = 0;
char* in_buff;
char* out_buff;
FILE* finput;
FILE* foutput;
size_t sizeCheck;
size_t inputBlockSize = FIO_blockID_to_blockSize(g_blockSizeId);
XXH32_state_t xxhState;
typedef size_t (*compressor_t) (void* dst, size_t dstSize, const void* src, size_t srcSize);
compressor_t compressor;
unsigned magicNumber;
/* Init */
XXH32_reset (&xxhState, FSE_CHECKSUM_SEED);
get_fileHandle(input_filename, output_filename, &finput, &foutput);
switch (g_compressor)
{
case FIO_fse:
compressor = FSE_compress;
magicNumber = FIO_magicNumber_fse;
break;
case FIO_huff0:
compressor = HUF_compress;
magicNumber = FIO_magicNumber_huff0;
break;
case FIO_zlibh:
compressor = FIO_ZLIBH_compress;
magicNumber = FIO_magicNumber_zlibh;
break;
default :
EXM_THROW(20, "unknown compressor selection");
}
/* Allocate Memory */
if (inputBlockSize==0) EXM_THROW(0, "impossible problem, to please static analyzer");
in_buff = (char*)malloc(inputBlockSize);
out_buff = (char*)malloc(FSE_compressBound(inputBlockSize) + 5);
if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory");
/* Write Frame Header */
FIO_writeLE32(out_buff, magicNumber);
out_buff[4] = (char)g_blockSizeId; /* Max Block Size descriptor */
sizeCheck = fwrite(out_buff, 1, FIO_FRAMEHEADERSIZE, foutput);
if (sizeCheck!=FIO_FRAMEHEADERSIZE) EXM_THROW(22, "Write error : cannot write header");
compressedfilesize += FIO_FRAMEHEADERSIZE;
/* Main compression loop */
while (1)
{
/* Fill input Buffer */
size_t cSize;
size_t inSize = fread(in_buff, (size_t)1, (size_t)inputBlockSize, finput);
if (inSize==0) break;
filesize += inSize;
XXH32_update(&xxhState, in_buff, inSize);
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20));
/* Compress Block */
cSize = compressor(out_buff + FIO_maxBlockHeaderSize, FSE_compressBound(inputBlockSize), in_buff, inSize);
if (FSE_isError(cSize)) EXM_THROW(23, "Compression error : %s ", FSE_getErrorName(cSize));
/* Write cBlock */
switch(cSize)
{
size_t headerSize;
case 0: /* raw */
if (inSize == inputBlockSize)
{
out_buff[0] = (BYTE)((bt_raw << 6) + BIT5);
headerSize = 1;
}
else
{
out_buff[2] = (BYTE)inSize;
out_buff[1] = (BYTE)(inSize >> 8);
out_buff[0] = (BYTE)(bt_raw << 6);
headerSize = 3;
}
sizeCheck = fwrite(out_buff, 1, headerSize, foutput);
if (sizeCheck!=headerSize) EXM_THROW(24, "Write error : cannot write block header");
sizeCheck = fwrite(in_buff, 1, inSize, foutput);
if (sizeCheck!=(size_t)(inSize)) EXM_THROW(25, "Write error : cannot write block");
compressedfilesize += inSize + headerSize;
break;
case 1: /* rle */
if (inSize == inputBlockSize)
{
out_buff[0] = (BYTE)((bt_rle << 6) + BIT5);
headerSize = 1;
}
else
{
out_buff[2] = (BYTE)inSize;
out_buff[1] = (BYTE)(inSize >> 8);
out_buff[0] = (BYTE)(bt_raw << 6);
headerSize = 3;
}
out_buff[headerSize] = in_buff[0];
sizeCheck = fwrite(out_buff, 1, headerSize+1, foutput);
if (sizeCheck!=(headerSize+1)) EXM_THROW(26, "Write error : cannot write rle block");
compressedfilesize += headerSize + 1;
break;
default : /* compressed */
if (inSize == inputBlockSize)
{
out_buff[2] = (BYTE)((bt_compressed << 6) + BIT5);
out_buff[3] = (BYTE)(cSize >> 8);
out_buff[4] = (BYTE)cSize;
headerSize = 3;
}
else
{
out_buff[0] = (BYTE)(bt_compressed << 6);
out_buff[1] = (BYTE)(inSize >> 8);
out_buff[2] = (BYTE)inSize;
out_buff[3] = (BYTE)(cSize >> 8);
out_buff[4] = (BYTE)cSize;
headerSize = FIO_maxBlockHeaderSize;
}
sizeCheck = fwrite(out_buff+(FIO_maxBlockHeaderSize-headerSize), 1, headerSize+cSize, foutput);
if (sizeCheck!=(headerSize+cSize)) EXM_THROW(27, "Write error : cannot write rle block");
compressedfilesize += headerSize + cSize;
break;
}
