std::string HiresTexture::GenBaseName(const u8* texture, size_t texture_size, const u8* tlut, size_t tlut_size, u32 width, u32 height, int format, bool has_mipmaps, bool dump) { std::string name = ""; HiresTextureCache::iterator convert_iter; // checking for min/max on paletted textures u32 min = 0xffff; u32 max = 0; switch (tlut_size) { case 0: break; case 16 * 2: for (size_t i = 0; i < texture_size; i++) { min = std::min<u32>(min, texture[i] & 0xf); min = std::min<u32>(min, texture[i] >> 4); max = std::max<u32>(max, texture[i] & 0xf); max = std::max<u32>(max, texture[i] >> 4); } break; case 256 * 2: for (size_t i = 0; i < texture_size; i++) { min = std::min<u32>(min, texture[i]); max = std::max<u32>(max, texture[i]); } break; case 16384 * 2: for (size_t i = 0; i < texture_size / 2; i++) { min = std::min<u32>(min, Common::swap16(((u16*)texture)[i]) & 0x3fff); max = std::max<u32>(max, Common::swap16(((u16*)texture)[i]) & 0x3fff); } break; } if (tlut_size > 0) { tlut_size = 2 * (max + 1 - min); tlut += 2 * min; } u64 tex_hash = XXH64(texture, texture_size, 0); u64 tlut_hash = 0; if (tlut_size) tlut_hash = XXH64(tlut, tlut_size, 0); std::string basename = s_format_prefix + StringFromFormat("%dx%d%s_%016" PRIx64, width, height, has_mipmaps ? "_m" : "", tex_hash); std::string tlutname = tlut_size ? StringFromFormat("_%016" PRIx64, tlut_hash) : ""; std::string formatname = StringFromFormat("_%d", format); std::string fullname = basename + tlutname + formatname; return fullname; }
unsigned long long michet_xxhash(const void* buffer, size_t length) { unsigned long long const seed = 31337; unsigned long long const hash = XXH64(buffer, length, seed); return hash; }
/* lookup key in hash table */ char * hash_table_lookup(hash_table_t *t, char *key, char **val, unsigned long *hash) { unsigned long index; char *prow; hash_table_row_hdr_t *phdr; char *pkey; char *pval; *hash = XXH64(key, t->keylen, 0); index = *hash % t->numbuckets; prow = t->buckets[index]; while (prow != NULL) { /* collison */ phdr = (hash_table_row_hdr_t *)prow; pkey = prow + sizeof(hash_table_row_hdr_t); pval = prow + sizeof(hash_table_row_hdr_t) + t->keylen; if (memcmp(pkey, key, t->keylen) == 0) { /* found key */ *val = pval; return prow; } else { /* keys do not match - try next item in list */ prow = phdr->hnext; } } return NULL; }
// Function handles the rehash process encountered when a hash reaches // 80% capacity. Hate locking the entire function, but we're rehashing, so it's // pretty much unavoidable. void rehash(hash_t *table) { // Acquire write-lock for hash. pthread_rwlock_wrlock(&table->lock); // Abort rehash if we're frozen. if (table->frozen) { pthread_rwlock_unlock(&table->lock); return; } // Allocate new table with calloc to allow for NULL checks. hash_node_t **new_data = calloc(table->size * 2, sizeof(hash_node_t *)); // Copy all previous data into new, larger, hash. hash_node_t **old_data = table->data; for(int i = 0; i < table->size; i++) { hash_node_t *current = old_data[i]; while (current) { hash_node_t *tmp = current->next; current->next = NULL; // Calculate new hash value and insert. unsigned int hash = (unsigned int) XXH64(current->key, strlen(current->key), 0) % (table->size * 2); new_data[hash] = insert_hash_node(new_data[hash], current); current = tmp; } } // Update hash struct with changes. table->data = new_data; table->size *= 2; free(old_data); }
static u32 computeHash(u32 address, u32 size) { #ifdef _M_X64 return XXH64(Memory::GetPointer(address), size, 0xBACD7814BACD7814LL); #else return XXH32(Memory::GetPointer(address), size, 0xBACD7814); #endif }
void MakeHash(std::enable_if_t<U::HasHash>* = 0) { m_hash = 0; m_hash ^= m_vertex.Hash(); m_hash ^= m_fragment.Hash(); m_hash ^= m_geometry.Hash(); m_hash ^= m_control.Hash(); m_hash ^= m_evaluation.Hash(); m_hash ^= XXH64(&m_additionalInfo, sizeof(m_additionalInfo), 0); }
static void xxHash64_test(const void *key, int len, uint32_t seed, void *out) { #if 0 if (! state) state = XXH64_createState (); XXH64_reset (state, seed); XXH64_update (state, key, len); *(uint64_t*)out = XXH64_digest (state); #else *(uint64_t*)out = XXH64 (key, len, seed); #endif }
static void rspamd_ucl_fin_cb (rspamd_mempool_t * pool, struct map_cb_data *data) { struct rspamd_ucl_map_cbdata *cbdata = data->cur_data, *prev = data->prev_data; ucl_object_t *obj; struct ucl_parser *parser; guint32 checksum; ucl_object_iter_t it = NULL; const ucl_object_t *cur; struct rspamd_config *cfg = data->map->cfg; if (prev != NULL) { if (prev->buf != NULL) { g_string_free (prev->buf, TRUE); } g_free (prev); } if (cbdata == NULL) { msg_err_config ("map fin error: new data is NULL"); return; } checksum = XXH64 (cbdata->buf->str, cbdata->buf->len, 0); if (data->map->checksum != checksum) { /* New data available */ parser = ucl_parser_new (0); if (!ucl_parser_add_chunk (parser, cbdata->buf->str, cbdata->buf->len)) { msg_err_config ("cannot parse map %s: %s", data->map->uri, ucl_parser_get_error (parser)); ucl_parser_free (parser); } else { obj = ucl_parser_get_object (parser); ucl_parser_free (parser); it = NULL; while ((cur = ucl_iterate_object (obj, &it, true))) { ucl_object_replace_key (cbdata->cfg->rcl_obj, (ucl_object_t *)cur, cur->key, cur->keylen, false); } ucl_object_unref (obj); data->map->checksum = checksum; } } else { msg_info_config ("do not reload map %s, checksum is the same: %d", data->map->uri, checksum); } }
/* insert element into hash table */ char * hash_table_insert(hash_table_t *t, char *key, char *val) { unsigned long hash, index; char *prow; hash_table_row_hdr_t *phdr; char *pkey; char *pval; hash = XXH64(key, t->keylen, 0); index = hash % t->numbuckets; prow = t->buckets[index]; while (prow != NULL) { /* collison */ phdr = (hash_table_row_hdr_t *)prow; pkey = prow + sizeof(hash_table_row_hdr_t); pval = prow + sizeof(hash_table_row_hdr_t) + t->keylen; if (memcmp(pkey, key, t->keylen) == 0) { /* same key - aggregate value */ t->aggr_callback(pkey, pval, val, t->callback_data); return prow; } else { /* keys do not match - try next item in list */ prow = phdr->next; } } /* new entry */ prow = malloc(sizeof(hash_table_row_hdr_t) + t->keylen + t->vallen); t->numentries++; if (prow == NULL) { return NULL; } phdr = (hash_table_row_hdr_t *)prow; pkey = prow + sizeof(hash_table_row_hdr_t); pval = prow + sizeof(hash_table_row_hdr_t) + t->keylen; memcpy(pkey, key, t->keylen); memcpy(pval, val, t->vallen); phdr->hash = hash; phdr->next = t->buckets[index]; t->buckets[index] = prow; return prow; }
STATIC mp_obj_t mod_eoslib_hash64(mp_obj_t obj1) { uint64_t key = 0; if (MP_OBJ_IS_STR_OR_BYTES(obj1)) { size_t len; const char* str = mp_obj_str_get_data(obj1, &len); key = XXH64(str, len, 0); return mp_obj_new_int_from_ll(key); } else if (MP_OBJ_IS_INT(obj1)) { return obj1; } else { mp_raise_TypeError("can't hash unsupported type"); return mp_const_none;//mute return type check } }
/* * Class: net_jpountz_xxhash_XXHashJNI * Method: XXH64BB * Signature: (Ljava/nio/ByteBuffer;IIJ)J */ JNIEXPORT jlong JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH64BB (JNIEnv *env, jclass cls, jobject buf, jint off, jint len, jlong seed) { char* in; jlong h64; in = (char*) (*env)->GetDirectBufferAddress(env, buf); if (in == NULL) { throw_OOM(env); return 0; } h64 = XXH64(in + off, len, seed); return h64; }
gboolean rspamd_bloom_add (rspamd_bloom_filter_t * bloom, const gchar *s) { size_t n, len; u_char t; guint v; if (s == NULL) { return FALSE; } len = strlen (s); for (n = 0; n < bloom->nfuncs; ++n) { v = XXH64 (s, len, bloom->seeds[n]) % bloom->asize; INCBIT (bloom->a, v, t); } return TRUE; }
/* * Class: net_jpountz_xxhash_XXHashJNI * Method: XXH64 * Signature: ([BIIJ)J */ JNIEXPORT jlong JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH64 (JNIEnv *env, jclass cls, jbyteArray buf, jint off, jint len, jlong seed) { char* in; jlong h64; in = (char*) (*env)->GetPrimitiveArrayCritical(env, buf, 0); if (in == NULL) { throw_OOM(env); return 0; } h64 = XXH64(in + off, len, seed); (*env)->ReleasePrimitiveArrayCritical(env, buf, in, 0); return h64; }
// Handle removal of a key from hash. int hash_drop(hash_t *table, char *key) { // Verify parameters. if (!table || table->count == 0 || !key) return HASH_INVAL_ERROR; // Acquire read lock for searching. pthread_rwlock_rdlock(&table->lock); // Generate hash value and find data. unsigned int hash = (unsigned int) XXH64(key, strlen(key), 0) % table->size; // Abort if we're frozen. if (table->frozen) { pthread_rwlock_unlock(&table->lock); return HASH_FROZEN_ERROR; } if (find_hash_node(table->data[hash], key)) { // We found it. Switch locks for writing. // Since we've only used a read-lock up to this point, it's possible for // multiple threads to attempt to delete the same key and all make it to this // point. Ensure that we're the first one here, and the only one to do the work // by checking for the key again. pthread_rwlock_unlock(&table->lock); pthread_rwlock_wrlock(&table->lock); if (!find_hash_node(table->data[hash], key)) { // Contention on key removal. pthread_rwlock_unlock(&table->lock); return HASH_NOTFOUND_ERROR; } // We have exclusive write access to the hash, and have verified that we're // either the first, or the only, thread trying to delete this key. table->data[hash] = remove_hash_node(table->data[hash], key, table->destruct); table->count--; pthread_rwlock_unlock(&table->lock); return HASH_SUCCESS; } else { // Key does not exist in table. pthread_rwlock_unlock(&table->lock); return HASH_NOTFOUND_ERROR; } }
// Function handles getting data out of a hash for a specific key. int hash_get(hash_t *table, char *key, void *buf) { // Verify parameters. if (!table || !table->count || !key || !buf) return HASH_INVAL_ERROR; // Acquire read-lock. pthread_rwlock_rdlock(&table->lock); // Generate hash value. unsigned int hash = (unsigned int) XXH64(key, strlen(key), 0) % table->size; // Find it. hash_node_t *found = find_hash_node(table->data[hash], key); // Return the data if it was found. if (found) { memcpy(buf, found->data, table->elem_size); pthread_rwlock_unlock(&table->lock); return HASH_SUCCESS; } else { pthread_rwlock_unlock(&table->lock); return HASH_NOTFOUND_ERROR; } }
int basicTests(U32 seed, double compressibility) { int testResult = 0; void* CNBuffer; void* compressedBuffer; void* decodedBuffer; U32 randState = seed; int cSize, testSize; LZ4F_preferences_t prefs; LZ4F_compressionContext_t cctx = NULL; U64 crcOrig; LZ4SG_in_t sg_cin [MAX_SG_BUFFERS]; LZ4SG_out_t sg_cout[MAX_SG_BUFFERS]; LZ4SG_in_t sg_din [MAX_SG_BUFFERS]; LZ4SG_out_t sg_dout[MAX_SG_BUFFERS]; size_t sg_in_len, sg_out_len; size_t maxDstSize = LZ4_SG_compressBound(COMPRESSIBLE_NOISE_LENGTH, NELEMS(sg_cin), NELEMS(sg_cout)); size_t sourceSizeOut; /* Create compressible test buffer */ memset(&prefs, 0, sizeof(prefs)); CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); compressedBuffer = malloc(maxDstSize); decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH + DECODE_GUARD_LENGTH); FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); /* Trivial tests : one input and one output buffers */ testSize = COMPRESSIBLE_NOISE_LENGTH; DISPLAYLEVEL(3, "One input and one output buffers : \n"); sg_cin[0].sg_base = CNBuffer; sg_cin[0].sg_len = COMPRESSIBLE_NOISE_LENGTH; sg_in_len = 1; sg_cout[0].sg_base = compressedBuffer; sg_cout[0].sg_len = maxDstSize; sg_out_len = 1; sourceSizeOut = testSize; cSize = LZ4_SG_compress(&sg_cin[0], sg_in_len, &sg_cout[0], sg_out_len, &sourceSizeOut, maxDstSize, DEFAULT_ACCEL); DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); UT_VERIFY(cSize > 0, goto _output_error); DISPLAYLEVEL(3, "Decompress test various valid sg_out and maxDstSize combinations \n"); { U64 crcDest; // sg_out.sg_len == maxDstSize == originalSize sg_din[0].sg_base = compressedBuffer; sg_din[0].sg_len = cSize; sg_dout[0].sg_base = decodedBuffer; sg_dout[0].sg_len = COMPRESSIBLE_NOISE_LENGTH; sourceSizeOut = cSize; maxDstSize = testSize; memset(decodedBuffer, 0, testSize); testResult = LZ4_SG_decompress(&sg_din[0], sg_in_len, &sg_dout[0], sg_out_len, &sourceSizeOut, maxDstSize); crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); DISPLAYLEVEL(3, "Decompressed %i bytes (out of %i bytes) to a %i bytes (out of %i bytes) dst_limit %i bytes\n", (int)sourceSizeOut, (int)cSize, (int)testResult, (int)testSize, (int)maxDstSize); UT_VERIFY(testResult > 0 , goto _output_error); UT_VERIFY(testResult == testSize, goto _output_error); UT_VERIFY( crcDest == crcOrig , goto _output_error); // maxDstSize > originalSize maxDstSize = COMPRESSIBLE_NOISE_LENGTH + DECODE_GUARD_LENGTH; sourceSizeOut = cSize; memset(decodedBuffer, 0, testSize); testResult = LZ4_SG_decompress(&sg_din[0], sg_in_len, &sg_dout[0], sg_out_len, &sourceSizeOut, maxDstSize); crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); DISPLAYLEVEL(3, "Decompressed %i bytes (out of %i bytes) to a %i bytes (out of %i bytes) dst_limit %i bytes\n", (int)sourceSizeOut, (int)cSize, (int)testResult, (int)testSize, (int)maxDstSize); UT_VERIFY(testResult > 0 , goto _output_error); UT_VERIFY(testResult == testSize, goto _output_error); UT_VERIFY( crcDest == crcOrig , goto _output_error); // sg_out.sg_len > originalSize maxDstSize = testSize; sg_dout[0].sg_len = COMPRESSIBLE_NOISE_LENGTH + DECODE_GUARD_LENGTH; sourceSizeOut = cSize; memset(decodedBuffer, 0, testSize); testResult = LZ4_SG_decompress(&sg_din[0], sg_in_len, &sg_dout[0], sg_out_len, &sourceSizeOut, maxDstSize); crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); DISPLAYLEVEL(3, "Decompressed %i bytes (out of %i bytes) to a %i bytes (out of %i bytes) dst_limit %i bytes\n", (int)sourceSizeOut, (int)cSize, (int)testResult, (int)testSize, (int)maxDstSize); UT_VERIFY(testResult > 0 , goto _output_error); UT_VERIFY(testResult == testSize, goto _output_error); UT_VERIFY( crcDest == crcOrig , goto _output_error); } DISPLAYLEVEL(3, "Frame Decompression test (%08x): \n", (unsigned int)crcOrig); { int lz4f_result = verify_basic_LZ4F_decompression(compressedBuffer, cSize, crcOrig, decodedBuffer, testSize); UT_VERIFY(0 == lz4f_result, goto _output_error); } // basic SGL test { // prepare SGL input const size_t num_data_buffers = 16; const size_t buf_size_bytes = 4 KB; unsigned int i; for (i = 0; i < 1+num_data_buffers; i++) { sg_cin [i].sg_base = malloc(buf_size_bytes); sg_cin [i].sg_len = buf_size_bytes; sg_cout[i].sg_base = malloc(buf_size_bytes); sg_cout[i].sg_len = buf_size_bytes; sg_din [i].sg_base = malloc(buf_size_bytes); sg_din [i].sg_len = buf_size_bytes; sg_dout[i].sg_base = malloc(buf_size_bytes); sg_dout[i].sg_len = buf_size_bytes; } for (i = 0; i < num_data_buffers; i++) { memcpy((void *)sg_cin [i].sg_base, ((const char *)CNBuffer)+(i*buf_size_bytes), buf_size_bytes); } sg_in_len = num_data_buffers; sg_out_len = 1+num_data_buffers; testSize = num_data_buffers * buf_size_bytes; maxDstSize = (1+num_data_buffers) * buf_size_bytes; crcOrig = XXH64(CNBuffer, testSize, 1); DISPLAYLEVEL(3, "Compress 16 4KB buffers into 17 4KB buffers : \n"); sourceSizeOut = testSize; cSize = LZ4_SG_compress(&sg_cin[0], sg_in_len, &sg_cout[0], sg_out_len, &sourceSizeOut, maxDstSize, DEFAULT_ACCEL); DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); UT_VERIFY(cSize > 0, goto _output_error); DISPLAYLEVEL(3, "Decompress 17 4KB buffers into 16 4KB buffers : \n"); for (i = 0; i < 1+num_data_buffers; i++) { memcpy((void *)sg_din[i].sg_base, sg_cout[i].sg_base, buf_size_bytes); } sourceSizeOut = cSize; maxDstSize = testSize; sg_in_len = 1+num_data_buffers; sg_out_len = num_data_buffers; memset(decodedBuffer, 0, testSize); testResult = LZ4_SG_decompress(&sg_din[0], sg_in_len, &sg_dout[0], sg_out_len, &sourceSizeOut, maxDstSize); DISPLAYLEVEL(3, "Decompressed %i bytes (out of %i bytes) to a %i bytes (out of %i bytes) dst_limit %i bytes\n", (int)sourceSizeOut, (int)cSize, (int)testResult, (int)testSize, (int)maxDstSize); for (i = 0; i < num_data_buffers; i++) { memcpy(((char *)decodedBuffer)+(i*buf_size_bytes), sg_dout[i].sg_base, buf_size_bytes); } UT_VERIFY(testResult > 0 , goto _output_error); UT_VERIFY(testResult == testSize, goto _output_error); U64 crcDest; crcDest = XXH64(decodedBuffer, testSize, 1); UT_VERIFY( crcDest == crcOrig , goto _output_error); DISPLAYLEVEL(3, "verify frame decompress on concatenated buffer: \n"); for (i = 0; i < 1+num_data_buffers; i++) { memcpy(((char *)compressedBuffer)+(i*buf_size_bytes), sg_cout[i].sg_base, buf_size_bytes); } int lz4f_result = verify_basic_LZ4F_decompression(compressedBuffer, cSize, crcOrig, decodedBuffer, testSize); UT_VERIFY(0 == lz4f_result, goto _output_error); // release SGL for (i = 0; i < 1+num_data_buffers; i++) { free((void *)sg_cin [i].sg_base); sg_cin [i].sg_base = NULL; sg_cin [i].sg_len = 0; free(sg_cout[i].sg_base); sg_cout[i].sg_base = NULL; sg_cout[i].sg_len = 0; free((void *)sg_din [i].sg_base); sg_din [i].sg_base = NULL; sg_din [i].sg_len = 0; free(sg_dout[i].sg_base); sg_dout[i].sg_base = NULL; sg_dout[i].sg_len = 0; } } DISPLAY("Basic tests completed \n"); testResult = 0; _end: free(CNBuffer); free(compressedBuffer); free(decodedBuffer); LZ4F_freeCompressionContext(cctx); cctx = NULL; return testResult; _output_error: testResult = 1; DISPLAY("Error detected ! \n"); locateBuffDiff(CNBuffer, decodedBuffer, testSize); goto _end; // unreachable return -1; }
gint rspamd_re_cache_compile_hyperscan (struct rspamd_re_cache *cache, const char *cache_dir, gdouble max_time, gboolean silent, GError **err) { g_assert (cache != NULL); g_assert (cache_dir != NULL); #ifndef WITH_HYPERSCAN g_set_error (err, rspamd_re_cache_quark (), EINVAL, "hyperscan is disabled"); return -1; #else GHashTableIter it, cit; gpointer k, v; struct rspamd_re_class *re_class; gchar path[PATH_MAX]; hs_database_t *test_db; gint fd, i, n, *hs_ids = NULL, pcre_flags, re_flags; guint64 crc; rspamd_regexp_t *re; hs_compile_error_t *hs_errors; guint *hs_flags = NULL; const gchar **hs_pats = NULL; gchar *hs_serialized; gsize serialized_len, total = 0; struct iovec iov[7]; g_hash_table_iter_init (&it, cache->re_classes); while (g_hash_table_iter_next (&it, &k, &v)) { re_class = v; rspamd_snprintf (path, sizeof (path), "%s%c%s.hs", cache_dir, G_DIR_SEPARATOR, re_class->hash); if (rspamd_re_cache_is_valid_hyperscan_file (cache, path, TRUE, TRUE)) { fd = open (path, O_RDONLY, 00600); /* Read number of regexps */ g_assert (fd != -1); lseek (fd, RSPAMD_HS_MAGIC_LEN + sizeof (cache->plt), SEEK_SET); read (fd, &n, sizeof (n)); close (fd); if (re_class->type_len > 0) { if (!silent) { msg_info_re_cache ( "skip already valid class %s(%*s) to cache %6s, %d regexps", rspamd_re_cache_type_to_string (re_class->type), (gint) re_class->type_len - 1, re_class->type_data, re_class->hash, n); } } else { if (!silent) { msg_info_re_cache ( "skip already valid class %s to cache %6s, %d regexps", rspamd_re_cache_type_to_string (re_class->type), re_class->hash, n); } } continue; } fd = open (path, O_CREAT|O_TRUNC|O_EXCL|O_WRONLY, 00600); if (fd == -1) { g_set_error (err, rspamd_re_cache_quark (), errno, "cannot open file " "%s: %s", path, strerror (errno)); return -1; } g_hash_table_iter_init (&cit, re_class->re); n = g_hash_table_size (re_class->re); hs_flags = g_malloc0 (sizeof (*hs_flags) * n); hs_ids = g_malloc (sizeof (*hs_ids) * n); hs_pats = g_malloc (sizeof (*hs_pats) * n); i = 0; while (g_hash_table_iter_next (&cit, &k, &v)) { re = v; pcre_flags = rspamd_regexp_get_pcre_flags (re); re_flags = rspamd_regexp_get_flags (re); if (re_flags & RSPAMD_REGEXP_FLAG_PCRE_ONLY) { /* Do not try to compile bad regexp */ msg_info_re_cache ( "do not try compile %s to hyperscan as it is PCRE only", rspamd_regexp_get_pattern (re)); continue; } hs_flags[i] = 0; #ifndef WITH_PCRE2 if (pcre_flags & PCRE_FLAG(UTF8)) { hs_flags[i] |= HS_FLAG_UTF8; } #else if (pcre_flags & PCRE_FLAG(UTF)) { hs_flags[i] |= HS_FLAG_UTF8; } #endif if (pcre_flags & PCRE_FLAG(CASELESS)) { hs_flags[i] |= HS_FLAG_CASELESS; } if (pcre_flags & PCRE_FLAG(MULTILINE)) { hs_flags[i] |= HS_FLAG_MULTILINE; } if (pcre_flags & PCRE_FLAG(DOTALL)) { hs_flags[i] |= HS_FLAG_DOTALL; } if (rspamd_regexp_get_maxhits (re) == 1) { hs_flags[i] |= HS_FLAG_SINGLEMATCH; } if (hs_compile (rspamd_regexp_get_pattern (re), hs_flags[i], cache->vectorized_hyperscan ? HS_MODE_VECTORED : HS_MODE_BLOCK, &cache->plt, &test_db, &hs_errors) != HS_SUCCESS) { msg_info_re_cache ("cannot compile %s to hyperscan, try prefilter match", rspamd_regexp_get_pattern (re)); hs_free_compile_error (hs_errors); /* The approximation operation might take a significant * amount of time, so we need to check if it's finite */ if (rspamd_re_cache_is_finite (cache, re, hs_flags[i], max_time)) { hs_flags[i] |= HS_FLAG_PREFILTER; hs_ids[i] = rspamd_regexp_get_cache_id (re); hs_pats[i] = rspamd_regexp_get_pattern (re); i++; } } else { hs_ids[i] = rspamd_regexp_get_cache_id (re); hs_pats[i] = rspamd_regexp_get_pattern (re); i ++; hs_free_database (test_db); } } /* Adjust real re number */ n = i; if (n > 0) { /* Create the hs tree */ if (hs_compile_multi (hs_pats, hs_flags, hs_ids, n, cache->vectorized_hyperscan ? HS_MODE_VECTORED : HS_MODE_BLOCK, &cache->plt, &test_db, &hs_errors) != HS_SUCCESS) { g_set_error (err, rspamd_re_cache_quark (), EINVAL, "cannot create tree of regexp when processing '%s': %s", hs_pats[hs_errors->expression], hs_errors->message); g_free (hs_flags); g_free (hs_ids); g_free (hs_pats); close (fd); hs_free_compile_error (hs_errors); return -1; } g_free (hs_pats); if (hs_serialize_database (test_db, &hs_serialized, &serialized_len) != HS_SUCCESS) { g_set_error (err, rspamd_re_cache_quark (), errno, "cannot serialize tree of regexp for %s", re_class->hash); close (fd); g_free (hs_ids); g_free (hs_flags); hs_free_database (test_db); return -1; } hs_free_database (test_db); /* * Magic - 8 bytes * Platform - sizeof (platform) * n - number of regexps * n * <regexp ids> * n * <regexp flags> * crc - 8 bytes checksum * <hyperscan blob> */ crc = XXH64 (hs_serialized, serialized_len, 0xdeadbabe); if (cache->vectorized_hyperscan) { iov[0].iov_base = (void *) rspamd_hs_magic_vector; } else { iov[0].iov_base = (void *) rspamd_hs_magic; } iov[0].iov_len = RSPAMD_HS_MAGIC_LEN; iov[1].iov_base = &cache->plt; iov[1].iov_len = sizeof (cache->plt); iov[2].iov_base = &n; iov[2].iov_len = sizeof (n); iov[3].iov_base = hs_ids; iov[3].iov_len = sizeof (*hs_ids) * n; iov[4].iov_base = hs_flags; iov[4].iov_len = sizeof (*hs_flags) * n; iov[5].iov_base = &crc; iov[5].iov_len = sizeof (crc); iov[6].iov_base = hs_serialized; iov[6].iov_len = serialized_len; if (writev (fd, iov, G_N_ELEMENTS (iov)) == -1) { g_set_error (err, rspamd_re_cache_quark (), errno, "cannot serialize tree of regexp to %s: %s", path, strerror (errno)); close (fd); g_free (hs_ids); g_free (hs_flags); g_free (hs_serialized); return -1; } if (re_class->type_len > 0) { msg_info_re_cache ( "compiled class %s(%*s) to cache %6s, %d regexps", rspamd_re_cache_type_to_string (re_class->type), (gint) re_class->type_len - 1, re_class->type_data, re_class->hash, n); } else { msg_info_re_cache ( "compiled class %s to cache %6s, %d regexps", rspamd_re_cache_type_to_string (re_class->type), re_class->hash, n); } total += n; g_free (hs_serialized); g_free (hs_ids); g_free (hs_flags); } close (fd); } return total; #endif }
int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration) { unsigned testResult = 0; unsigned testNb = 0; void* srcBuffer = NULL; void* compressedBuffer = NULL; void* decodedBuffer = NULL; U32 coreRand = seed; LZ4F_decompressionContext_t dCtx = NULL; LZ4F_compressionContext_t cCtx = NULL; size_t result; const U32 startTime = FUZ_GetMilliStart(); XXH64_state_t xxh64; # define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } // backup all allocated addresses, from which we will later select buffers const size_t max_buf_size = 131 KB; size_t num_buf_size_distribution_deviations = 0; LZ4SG_in_t sg_in_buf_potential [2*MAX_SG_BUFFERS]; LZ4SG_out_t sg_out_buf_potential[2*MAX_SG_BUFFERS]; LZ4SG_in_t sg_cin [MAX_SG_BUFFERS]; LZ4SG_out_t sg_cout[MAX_SG_BUFFERS]; LZ4SG_in_t sg_din [MAX_SG_BUFFERS]; LZ4SG_out_t sg_dout[MAX_SG_BUFFERS]; size_t sg_cin_len, sg_cout_len, sg_din_len, sg_dout_len; const size_t maxDstSize = LZ4_SG_compressBound(srcDataLength, NELEMS(sg_cin), NELEMS(sg_cout)); unsigned int i; for (i = 0; i < NELEMS(sg_in_buf_potential); i++) { sg_in_buf_potential [i].sg_base = malloc(max_buf_size); sg_in_buf_potential [i].sg_len = max_buf_size; sg_out_buf_potential[i].sg_base = malloc(max_buf_size); sg_out_buf_potential[i].sg_len = max_buf_size; } /* Init */ duration *= 1000; /* Create buffers */ result = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result); result = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION); CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result); srcBuffer = malloc(srcDataLength); CHECK(srcBuffer==NULL, "srcBuffer Allocation failed"); const size_t compressedBufferLength = maxDstSize; compressedBuffer = malloc(compressedBufferLength); CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed"); decodedBuffer = calloc(1, srcDataLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */ CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed"); FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand); /* jump to requested testNb */ for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); // sync randomizer /* main fuzzer test loop */ for ( ; (testNb < nbTests) || (duration > FUZ_GetMilliSpan(startTime)) ; testNb++) { U32 randState = coreRand ^ prime1; (void)FUZ_rand(&coreRand); /* update seed */ srand48(FUZ_rand(&randState)); DISPLAYUPDATE(2, "\r%5u ", testNb); const size_t max_src_buf_size = (4 MB > srcDataLength) ? srcDataLength : 4 MB; unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(max_src_buf_size-1) - 1)) + 1; const size_t min_src_size = 20; const size_t min_first_dest_buf_size = 21; const size_t min_src_buf_size = 1; const size_t min_dst_buf_size = 10; size_t srcSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + min_src_size; size_t srcStart = FUZ_rand(&randState) % (srcDataLength - srcSize); size_t cSize; size_t dstSize; size_t dstSizeBound; U64 crcOrig, crcDecoded; unsigned int test_selection = FUZ_rand(&randState); //TODO: enable lz4f_compress_compatibility_test with LZ4_SG_decompress int lz4f_compress_compatibility_test = 0;//(test_selection % 4) == 0; if (!lz4f_compress_compatibility_test) { // SGL compress unsigned int buffer_selection = FUZ_rand(&randState); if ((buffer_selection & 0xF) == 1) { // SG compress single source and single target buffers sg_cin[0].sg_base = (BYTE*)srcBuffer+srcStart; sg_cin[0].sg_len = srcSize; sg_cin_len = 1; sg_cout[0].sg_base = compressedBuffer; sg_cout[0].sg_len = compressedBufferLength; sg_cout_len = 1; dstSizeBound = dstSize = compressedBufferLength; } else { // SG compress random number and size source and target buffers sg_cin_len = 1 + (FUZ_rand(&randState) % MAX_SG_BUFFERS); sg_cout_len = 1 + (FUZ_rand(&randState) % MAX_SG_BUFFERS); // single source buffer if (1 == sg_cin_len) { sg_cin[0].sg_base = (BYTE*)srcBuffer+srcStart; sg_cin[0].sg_len = srcSize; DISPLAYUPDATE(4, "INFO: single source buf size %i\n", (int)srcSize); } else { // multiple source buffers if (srcSize > sg_cin_len*max_buf_size/2) { srcSize = sg_cin_len*max_buf_size/2; num_buf_size_distribution_deviations++; DISPLAYUPDATE(4, "NOTE: source buffer total size deviation %i\n", (int)num_buf_size_distribution_deviations); } size_t exact_src_size = 0; unsigned int buf_size_mean = srcSize / sg_cin_len; for (i = 0; i < sg_cin_len; i++) { size_t buf_size = rnd_exponential(buf_size_mean, min_src_buf_size, max_buf_size); DISPLAYUPDATE(4, "INFO: source buf %i size %i\n", i, (int)buf_size); if (srcStart+exact_src_size+buf_size > srcDataLength) { buf_size = srcDataLength-(srcStart+exact_src_size); } sg_cin[i].sg_base = sg_in_buf_potential[i*2+1].sg_base; sg_cin[i].sg_len = buf_size; memcpy((void *)sg_cin[i].sg_base, (BYTE*)srcBuffer+srcStart+exact_src_size, buf_size); exact_src_size += buf_size; if (srcStart+exact_src_size == srcDataLength) { num_buf_size_distribution_deviations++; sg_cin_len = i+1; DISPLAYUPDATE(4, "NOTE: final source buffer size deviation %i (buffers number limited to %i)\n", (int)num_buf_size_distribution_deviations, (int)sg_cin_len); } } srcSize = exact_src_size; } // we can now derive the required limit for output dstSizeBound = LZ4_SG_compressBound(srcSize, sg_cin_len, sg_cout_len); // single target buffer if (1 == sg_cout_len) { sg_cout[0].sg_base = compressedBuffer; sg_cout[0].sg_len = compressedBufferLength; } else { // multiple target buffers int finalBufferTruncated = 0; dstSize = 0; unsigned int buf_size_mean = dstSizeBound / sg_cout_len; for (i = 0; i < sg_cout_len; i++) { const size_t min_buf_size = (i == 0) ? min_first_dest_buf_size : min_dst_buf_size; size_t buf_size = rnd_exponential(buf_size_mean, min_buf_size, max_buf_size); DISPLAYUPDATE(4, "INFO: target buf %i size %i\n", (int)i, (int)buf_size); if (dstSize+buf_size > dstSizeBound) { buf_size = dstSizeBound-dstSize; finalBufferTruncated = 1; } dstSize += buf_size; sg_cout[i].sg_base = sg_out_buf_potential[i*2+1].sg_base; sg_cout[i].sg_len = buf_size; if (finalBufferTruncated) { num_buf_size_distribution_deviations++; if (buf_size < min_buf_size) { // merge truncated with previous? if (i > 0) { sg_cout[i-1].sg_len += buf_size; if (sg_cout[i-1].sg_len > max_buf_size) { // skip, too much hassle DISPLAYUPDATE(4, "NOTE: unable to truncate final target buffer size (deviations %i), skipping\n", (int)num_buf_size_distribution_deviations); sg_cout_len = 0; break; } } else { // can this happen? DISPLAYUPDATE(4, "NOTE: unable to truncate first and final target buffer size (deviations %i), skipping\n", (int)num_buf_size_distribution_deviations); sg_cout_len = 0; break; } sg_cout_len = i; } else { sg_cout_len = i+1; } DISPLAYUPDATE(4, "NOTE: final target buffer size truncated (%i), buffers number limited to %i, final's size is now %i (deviations %i)\n", (int)buf_size, (int)sg_cout_len, (int)sg_cout[sg_cout_len-1].sg_len, (int)num_buf_size_distribution_deviations); } } // skip/abort condition if (0 == sg_cout_len) continue; } if ((buffer_selection & 0xF) == 0) { //TODO: select a random input and output buffer and split it in two, // feeding consecutive addresses as consecutive entries in SGL } } crcOrig = XXH64((BYTE*)srcBuffer+srcStart, srcSize, 1); size_t sourceSizeOut = srcSize; result = LZ4_SG_compress(&sg_cin[0], sg_cin_len, &sg_cout[0], sg_cout_len, &sourceSizeOut, maxDstSize, DEFAULT_ACCEL); if (((result == 0) || (sourceSizeOut != srcSize)) && (dstSize < dstSizeBound)) { // forgive compression failure when output total size is lower than bound num_buf_size_distribution_deviations++; DISPLAYUPDATE(4, "NOTE: dstSize %i < %i dstSizeBound, compression attempt failed, not totally unexpected (deviations %i), skipping\n", (int)dstSize, (int)dstSizeBound, (int)num_buf_size_distribution_deviations); continue; } CHECK(result <= 0, "Compression failed (error %i)", (int)result); CHECK(sourceSizeOut != srcSize, "Compression stopped at %i out of %i", (int)sourceSizeOut, (int)srcSize); cSize = result; } else { // LZ4F compression - use it in order to verify SGL decompress compatibility with it DISPLAYUPDATE(4, "INFO: LZ4F compression\n"); // alternative // size_t dstMaxSize = LZ4F_compressFrameBound(srcSize, prefsPtr); // DISPLAYLEVEL(3, "compressFrame srcSize %zu dstMaxSize %zu\n", // srcSize, dstMaxSize); // cSize = LZ4F_compressFrame(compressedBuffer, dstMaxSize, (char*)srcBuffer + srcStart, srcSize, prefsPtr); // CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize)); crcOrig = XXH64((BYTE*)srcBuffer+srcStart, srcSize, 1); unsigned BSId = 4 + (FUZ_rand(&randState) & 3); unsigned BMId = FUZ_rand(&randState) & 1; unsigned CCflag = FUZ_rand(&randState) & 1; unsigned autoflush = (FUZ_rand(&randState) & 7) == 2; U64 frameContentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0; LZ4F_preferences_t prefs; LZ4F_compressOptions_t cOptions; LZ4F_preferences_t* prefsPtr = &prefs; memset(&prefs, 0, sizeof(prefs)); memset(&cOptions, 0, sizeof(cOptions)); prefs.frameInfo.blockMode = (LZ4F_blockMode_t)BMId; prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)BSId; prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)CCflag; prefs.frameInfo.contentSize = frameContentSize; prefs.autoFlush = autoflush; prefs.compressionLevel = FUZ_rand(&randState) % 5; if ((FUZ_rand(&randState) & 0xF) == 1) prefsPtr = NULL; const BYTE* ip = (const BYTE*)srcBuffer + srcStart; const BYTE* const iend = ip + srcSize; BYTE* op = (BYTE*)compressedBuffer; BYTE* const oend = op + LZ4F_compressFrameBound(srcDataLength, NULL); unsigned maxBits = FUZ_highbit((U32)srcSize); result = LZ4F_compressBegin(cCtx, op, oend-op, prefsPtr); CHECK(LZ4F_isError(result), "Compression header failed (error %i)", (int)result); op += result; while (ip < iend) { unsigned nbBitsSeg = FUZ_rand(&randState) % maxBits; size_t iSize = (FUZ_rand(&randState) & ((1<<nbBitsSeg)-1)) + 1; size_t oSize = LZ4F_compressBound(iSize, prefsPtr); unsigned forceFlush = ((FUZ_rand(&randState) & 3) == 1); if (iSize > (size_t)(iend-ip)) iSize = iend-ip; cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1); DISPLAYLEVEL(3, "compressUpdate ip %d iSize %zu oSize %zu forceFlush %d\n", (int)(ip-((const BYTE*)srcBuffer + srcStart)), iSize, oSize, forceFlush); result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions); CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result); op += result; ip += iSize; if (forceFlush) { result = LZ4F_flush(cCtx, op, oend-op, &cOptions); CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result); op += result; } } result = LZ4F_compressEnd(cCtx, op, oend-op, &cOptions); CHECK(LZ4F_isError(result), "Compression completion failed (error %i)", (int)result); op += result; cSize = op-(BYTE*)compressedBuffer; } //DECOMPRESS test_selection = FUZ_rand(&randState); if (lz4f_compress_compatibility_test || ((test_selection % 2) == 0)) { //TODO: SGL decompress with random buffer sizes // SGL decompress with same buffer sizes used for compression // prepare din with cout's data sg_din_len = sg_cout_len; for (i = 0; i < sg_din_len; i++) { sg_din[i].sg_len = sg_cout[i].sg_len; if (sg_cout[i].sg_len <= max_buf_size) { // enough room to copy - do it sg_din[i].sg_base = sg_in_buf_potential[i*2+0].sg_base; if (sg_din[i].sg_base != sg_cout[i].sg_base) { memcpy((void *)sg_din[i].sg_base, sg_cout[i].sg_base, sg_cout[i].sg_len); } } else { // this is probably single output buffer - skip copy, use directly sg_din[i].sg_base = sg_cout[i].sg_base; } } // prepare dout to receive decompressed data sg_dout_len = sg_cin_len; for (i = 0; i < sg_dout_len; i++) { sg_dout[i].sg_len = sg_cin[i].sg_len; if (sg_cin[i].sg_len <= max_buf_size) { // enough room to decompress into independent buffer sg_dout[i].sg_base = sg_out_buf_potential[i*2+0].sg_base; } else { // this is probably single input buffer, use an external output buffer sg_dout[i].sg_base = decodedBuffer; } } size_t sourceSizeOut = cSize; size_t maxOutputSize = srcSize; int decomp_result = LZ4_SG_decompress(&sg_din[0], sg_din_len, &sg_dout[0], sg_dout_len, &sourceSizeOut, maxOutputSize); CHECK(decomp_result <= 0, "SG decompression failed (error %i)", (int)decomp_result); CHECK(decomp_result != (int)srcSize, "SG decompression stopped at %i", (int)decomp_result); // verify result checksum size_t total_checked = 0; XXH64_reset(&xxh64, 1); for (i = 0; (i < sg_dout_len) && ((int)total_checked < decomp_result); i++) { size_t cur_size = sg_dout[i].sg_len; size_t rem = decomp_result - total_checked; if (rem < cur_size) cur_size = rem; total_checked += cur_size; XXH64_update(&xxh64, sg_dout[i].sg_base, cur_size); } crcDecoded = XXH64_digest(&xxh64); if (crcDecoded != crcOrig) { DISPLAYLEVEL(1, "checked %i out of %i (crcDecoded %08x, crcOrig %08x)\n", (int)total_checked, decomp_result, (unsigned)crcDecoded, (unsigned)crcOrig); // locate error if any total_checked = 0; for (i = 0; (i < sg_dout_len) && ((int)total_checked < decomp_result); i++) { size_t cur_size = sg_dout[i].sg_len; size_t rem = decomp_result - total_checked; if (rem < cur_size) cur_size = rem; total_checked += cur_size; U64 crc_in = XXH64(sg_cin [i].sg_base, cur_size, 1); U64 crc_out = XXH64(sg_dout[i].sg_base, cur_size, 1); if (crc_in != crc_out) { locateBuffDiff(sg_cin[i].sg_base, sg_dout[i].sg_base, cur_size); break; } } DISPLAYLEVEL(1, "checked %i out of %i\n", (int)total_checked, decomp_result); } CHECK(crcDecoded != crcOrig, "Decompression corruption"); } else { // prepare compressedBuffer from SGL size_t total_copied = 0; for (i = 0; i < sg_cout_len; i++) { size_t buf_size_bytes = cSize - total_copied; if (buf_size_bytes == 0) break; if (buf_size_bytes > sg_cout[i].sg_len) buf_size_bytes = sg_cout[i].sg_len; if (((char *)compressedBuffer)+total_copied != sg_cout[i].sg_base) { memcpy(((char *)compressedBuffer)+total_copied, sg_cout[i].sg_base, buf_size_bytes); } total_copied += buf_size_bytes; } LZ4F_decompressOptions_t dOptions; memset(&dOptions, 0, sizeof(dOptions)); const BYTE* ip = (const BYTE*)compressedBuffer; const BYTE* const iend = ip + cSize; BYTE* op = (BYTE*)decodedBuffer; BYTE* const oend = op + srcDataLength; size_t totalOut = 0; unsigned maxBits = FUZ_highbit((U32)cSize); XXH64_reset(&xxh64, 1); if (maxBits < 3) maxBits = 3; while (ip < iend) { unsigned nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1; unsigned nbBitsO = (FUZ_rand(&randState) % (maxBits)) + 1; size_t iSize = (FUZ_rand(&randState) & ((1<<nbBitsI)-1)) + 1; size_t oSize = (FUZ_rand(&randState) & ((1<<nbBitsO)-1)) + 2; if (iSize > (size_t)(iend-ip)) iSize = iend-ip; if (oSize > (size_t)(oend-op)) oSize = oend-op; dOptions.stableDst = FUZ_rand(&randState) & 1; result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions); if (result == (size_t)-LZ4F_ERROR_contentChecksum_invalid) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize); CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s ip %d)", (int)result, LZ4F_getErrorName((LZ4F_errorCode_t)result), (int)(ip-(const BYTE*)compressedBuffer)); XXH64_update(&xxh64, op, (U32)oSize); totalOut += oSize; op += oSize; ip += iSize; } CHECK(result != 0, "Frame decompression failed (error %i)", (int)result); if (totalOut) /* otherwise, it's a skippable frame */ { crcDecoded = XXH64_digest(&xxh64); if (crcDecoded != crcOrig) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize); CHECK(crcDecoded != crcOrig, "Decompression corruption"); } } } DISPLAYLEVEL(2, "\rAll tests completed \n"); _end: LZ4F_freeDecompressionContext(dCtx); LZ4F_freeCompressionContext(cCtx); free(srcBuffer); free(compressedBuffer); free(decodedBuffer); for (i = 0; i < NELEMS(sg_in_buf_potential); i++) { free((void *)(sg_in_buf_potential [i].sg_base)); free( sg_out_buf_potential[i].sg_base); } if (num_buf_size_distribution_deviations > 0) { DISPLAYLEVEL(2, "NOTE: %i buffer size deviations \n", (int)num_buf_size_distribution_deviations); } if (pause) { DISPLAY("press enter to finish \n"); (void)getchar(); } return testResult; _output_error: testResult = 1; goto _end; // unreachable return -1; #undef CHECK }
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility) { BYTE* cNoiseBuffer[5]; BYTE* srcBuffer; size_t srcBufferSize = (size_t)1<<maxSrcLog; BYTE* copyBuffer; size_t copyBufferSize = srcBufferSize + (1<<maxSampleLog); BYTE* cBuffer; size_t cBufferSize = ZSTD_compressBound(srcBufferSize); BYTE* dstBuffer; size_t dstBufferSize = srcBufferSize; U32 result = 0; U32 testNb = 0; U32 coreSeed = seed, lseed = 0; ZBUFF_CCtx* zc; ZBUFF_DCtx* zd; XXH64_state_t crc64; U32 startTime = FUZ_GetMilliStart(); /* allocation */ zc = ZBUFF_createCCtx(); zd = ZBUFF_createDCtx(); cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize); copyBuffer= (BYTE*)malloc (copyBufferSize); dstBuffer = (BYTE*)malloc (dstBufferSize); cBuffer = (BYTE*)malloc (cBufferSize); CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] || !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd, "Not enough memory, fuzzer tests cancelled"); /* Create initial samples */ RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */ RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */ RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed); RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */ RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */ srcBuffer = cNoiseBuffer[2]; memset(copyBuffer, 0x65, copyBufferSize); memcpy(copyBuffer, srcBuffer, MIN(copyBufferSize,srcBufferSize)); /* make copyBuffer considered initialized */ /* catch up testNb */ for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed); /* test loop */ for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ ) { size_t sampleSize, sampleStart; size_t cSize; size_t maxTestSize, totalTestSize, readSize, totalCSize, genSize, totalGenSize; size_t errorCode; U32 sampleSizeLog, buffNb, n, nbChunks; U64 crcOrig, crcDest; /* init */ DISPLAYUPDATE(2, "\r%6u", testNb); if (nbTests >= testNb) DISPLAYUPDATE(2, "/%6u ", nbTests); FUZ_rand(&coreSeed); lseed = coreSeed ^ prime1; buffNb = FUZ_rand(&lseed) & 127; if (buffNb & 7) buffNb=2; /* select buffer */ else { buffNb >>= 3; if (buffNb & 7) { const U32 tnb[2] = { 1, 3 }; buffNb = tnb[buffNb >> 3]; } else { const U32 tnb[2] = { 0, 4 }; buffNb = tnb[buffNb >> 3]; } } srcBuffer = cNoiseBuffer[buffNb]; /* Multi - segments compression test */ XXH64_reset(&crc64, 0); nbChunks = (FUZ_rand(&lseed) & 127) + 2; sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog; maxTestSize = (size_t)1 << sampleSizeLog; maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1); ZBUFF_compressInit(zc, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1); totalTestSize = 0; cSize = 0; for (n=0; n<nbChunks; n++) { sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); readSize = sampleSize; /* random size output buffer */ sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); genSize = MIN (cBufferSize - cSize, sampleSize); errorCode = ZBUFF_compressContinue(zc, cBuffer+cSize, &genSize, srcBuffer+sampleStart, &readSize); CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode)); XXH64_update(&crc64, srcBuffer+sampleStart, readSize); memcpy(copyBuffer+totalTestSize, srcBuffer+sampleStart, readSize); cSize += genSize; totalTestSize += readSize; if ((FUZ_rand(&lseed) & 15) == 0) { /* add a few random flushes operations, to mess around */ sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); genSize = MIN (cBufferSize - cSize, sampleSize); errorCode = ZBUFF_compressFlush(zc, cBuffer+cSize, &genSize); CHECK (ZBUFF_isError(errorCode), "flush error : %s", ZBUFF_getErrorName(errorCode)); cSize += genSize; } if (totalTestSize > maxTestSize) break; } genSize = cBufferSize - cSize; errorCode = ZBUFF_compressEnd(zc, cBuffer+cSize, &genSize); CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode)); CHECK (errorCode != 0, "frame epilogue not fully consumed"); cSize += genSize; crcOrig = XXH64_digest(&crc64); /* multi - fragments decompression test */ ZBUFF_decompressInit(zd); totalCSize = 0; totalGenSize = 0; while (totalCSize < cSize) { sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); readSize = sampleSize; sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); genSize = MIN(sampleSize, dstBufferSize - totalGenSize); errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize); CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode)); totalGenSize += genSize; totalCSize += readSize; } CHECK (errorCode != 0, "frame not fully decoded"); CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size") CHECK (totalCSize != cSize, "compressed data should be fully read") crcDest = XXH64(dstBuffer, totalTestSize, 0); if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); CHECK (crcDest!=crcOrig, "decompressed data corrupted"); /* noisy/erroneous src decompression test */ /* add some noise */ nbChunks = (FUZ_rand(&lseed) & 7) + 2; for (n=0; n<nbChunks; n++) { size_t cStart; sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); if (sampleSize > cSize/3) sampleSize = cSize/3; sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); cStart = FUZ_rand(&lseed) % (cSize - sampleSize); memcpy(cBuffer+cStart, srcBuffer+sampleStart, sampleSize); } /* try decompression on noisy data */ ZBUFF_decompressInit(zd); totalCSize = 0; totalGenSize = 0; while ( (totalCSize < cSize) && (totalGenSize < dstBufferSize) ) { sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); readSize = sampleSize; sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); genSize = MIN(sampleSize, dstBufferSize - totalGenSize); errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize); if (ZBUFF_isError(errorCode)) break; /* error correctly detected */ totalGenSize += genSize; totalCSize += readSize; } }
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility) { BYTE* cNoiseBuffer[5]; BYTE* srcBuffer; BYTE* cBuffer; BYTE* dstBuffer; BYTE* mirrorBuffer; size_t srcBufferSize = (size_t)1<<maxSrcLog; size_t dstBufferSize = (size_t)1<<maxSampleLog; size_t cBufferSize = ZSTD_compressBound(dstBufferSize); U32 result = 0; U32 testNb = 0; U32 coreSeed = seed, lseed = 0; ZSTD_CCtx* refCtx; ZSTD_CCtx* ctx; ZSTD_DCtx* dctx; U32 startTime = FUZ_GetMilliStart(); /* allocation */ refCtx = ZSTD_createCCtx(); ctx = ZSTD_createCCtx(); dctx= ZSTD_createDCtx(); cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize); dstBuffer = (BYTE*)malloc (dstBufferSize); mirrorBuffer = (BYTE*)malloc (dstBufferSize); cBuffer = (BYTE*)malloc (cBufferSize); CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] || !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx, "Not enough memory, fuzzer tests cancelled"); /* Create initial samples */ RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */ RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */ RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed); RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */ RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */ srcBuffer = cNoiseBuffer[2]; /* catch up testNb */ for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed); /* test loop */ for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ ) { size_t sampleSize, sampleStart, maxTestSize, totalTestSize; size_t cSize, dSize, dSupSize, errorCode, totalCSize, totalGenSize; U32 sampleSizeLog, buffNb, cLevelMod, nbChunks, n; XXH64_CREATESTATE_STATIC(xxh64); U64 crcOrig, crcDest; int cLevel; BYTE* sampleBuffer; const BYTE* dict; size_t dictSize; /* init */ if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); } else { DISPLAYUPDATE(2, "\r%6u ", testNb); } FUZ_rand(&coreSeed); lseed = coreSeed ^ prime1; buffNb = FUZ_rand(&lseed) & 127; if (buffNb & 7) buffNb=2; else { buffNb >>= 3; if (buffNb & 7) { const U32 tnb[2] = { 1, 3 }; buffNb = tnb[buffNb >> 3]; } else { const U32 tnb[2] = { 0, 4 }; buffNb = tnb[buffNb >> 3]; } } srcBuffer = cNoiseBuffer[buffNb]; sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); /* create sample buffer (to catch read error with valgrind & sanitizers) */ sampleBuffer = (BYTE*)malloc(sampleSize); CHECK (sampleBuffer==NULL, "not enough memory for sample buffer"); memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); crcOrig = XXH64(sampleBuffer, sampleSize, 0); /* compression test */ cLevelMod = MAX(1, 38 - (int)(MAX(9, sampleSizeLog) * 2)); /* use high compression levels with small samples, for speed */ cLevel = (FUZ_rand(&lseed) % cLevelMod) +1; cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel); CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed"); /* compression failure test : too small dest buffer */ if (cSize > 3) { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ const size_t tooSmallSize = cSize - missing; static const U32 endMark = 0x4DC2B1A9; U32 endCheck; memcpy(dstBuffer+tooSmallSize, &endMark, 4); errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel); CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize); memcpy(&endCheck, dstBuffer+tooSmallSize, 4); CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); } /* successfull decompression tests*/ dSupSize = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1; dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize); CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize); crcDest = XXH64(dstBuffer, sampleSize, 0); CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(sampleBuffer, dstBuffer, sampleSize), (U32)sampleSize); free(sampleBuffer); /* no longer useful after this point */ /* truncated src decompression test */ { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ const size_t tooSmallSize = cSize - missing; void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch overflows */ CHECK(cBufferTooSmall == NULL, "not enough memory !"); memcpy(cBufferTooSmall, cBuffer, tooSmallSize); errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize); CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)"); free(cBufferTooSmall); } /* too small dst decompression test */ if (sampleSize > 3) { const size_t missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ const size_t tooSmallSize = sampleSize - missing; static const BYTE token = 0xA9; dstBuffer[tooSmallSize] = token; errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize); CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (U32)errorCode, (U32)tooSmallSize); CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow"); } /* noisy src decompression test */ if (cSize > 6) { const U32 maxNbBits = FUZ_highbit32((U32)(cSize-4)); size_t pos = 4; /* preserve magic number (too easy to detect) */ U32 nbBits = FUZ_rand(&lseed) % maxNbBits; size_t mask = (1<<nbBits) - 1; size_t skipLength = FUZ_rand(&lseed) & mask; pos += skipLength; while (pos < cSize) { /* add noise */ size_t noiseStart, noiseLength; nbBits = FUZ_rand(&lseed) % maxNbBits; if (nbBits>0) nbBits--; mask = (1<<nbBits) - 1; noiseLength = (FUZ_rand(&lseed) & mask) + 1; if ( pos+noiseLength > cSize ) noiseLength = cSize-pos; noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength); memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength); pos += noiseLength; /* keep some original src */ nbBits = FUZ_rand(&lseed) % maxNbBits; mask = (1<<nbBits) - 1; skipLength = FUZ_rand(&lseed) & mask; pos += skipLength; } /* decompress noisy source */ { U32 noiseSrc = FUZ_rand(&lseed) % 5; const U32 endMark = 0xA9B1C3D6; U32 endCheck; srcBuffer = cNoiseBuffer[noiseSrc]; memcpy(dstBuffer+sampleSize, &endMark, 4); errorCode = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize); /* result *may* be an unlikely success, but even then, it must strictly respect dest buffer boundaries */ CHECK((!ZSTD_isError(errorCode)) && (errorCode>sampleSize), "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (U32)errorCode, (U32)sampleSize); memcpy(&endCheck, dstBuffer+sampleSize, 4); CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow"); } } /* Streaming compression of scattered segments test */ XXH64_reset(xxh64, 0); nbChunks = (FUZ_rand(&lseed) & 127) + 2; sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog; maxTestSize = (size_t)1 << sampleSizeLog; maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1); if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1; sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); dict = srcBuffer + sampleStart; dictSize = sampleSize; errorCode = ZSTD_compressBegin(refCtx, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1); CHECK (ZSTD_isError(errorCode), "start streaming error : %s", ZSTD_getErrorName(errorCode)); errorCode = ZSTD_compress_insertDictionary(refCtx, dict, dictSize); CHECK (ZSTD_isError(errorCode), "dictionary insertion error : %s", ZSTD_getErrorName(errorCode)); errorCode = ZSTD_duplicateCCtx(ctx, refCtx); CHECK (ZSTD_isError(errorCode), "context duplication error : %s", ZSTD_getErrorName(errorCode)); totalTestSize = 0; cSize = 0; for (n=0; n<nbChunks; n++) { sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); if (cBufferSize-cSize < ZSTD_compressBound(sampleSize)) /* avoid invalid dstBufferTooSmall */ break; if (totalTestSize+sampleSize > maxTestSize) break; errorCode = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+sampleStart, sampleSize); CHECK (ZSTD_isError(errorCode), "multi-segments compression error : %s", ZSTD_getErrorName(errorCode)); cSize += errorCode; XXH64_update(xxh64, srcBuffer+sampleStart, sampleSize); memcpy(mirrorBuffer + totalTestSize, srcBuffer+sampleStart, sampleSize); totalTestSize += sampleSize; } errorCode = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize); CHECK (ZSTD_isError(errorCode), "multi-segments epilogue error : %s", ZSTD_getErrorName(errorCode)); cSize += errorCode; crcOrig = XXH64_digest(xxh64); /* streaming decompression test */ errorCode = ZSTD_resetDCtx(dctx); CHECK (ZSTD_isError(errorCode), "cannot init DCtx : %s", ZSTD_getErrorName(errorCode)); ZSTD_decompress_insertDictionary(dctx, dict, dictSize); totalCSize = 0; totalGenSize = 0; while (totalCSize < cSize) { size_t inSize = ZSTD_nextSrcSizeToDecompress(dctx); size_t genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize); CHECK (ZSTD_isError(genSize), "streaming decompression error : %s", ZSTD_getErrorName(genSize)); totalGenSize += genSize; totalCSize += inSize; } CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded"); CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size") CHECK (totalCSize != cSize, "compressed data should be fully read") crcDest = XXH64(dstBuffer, totalTestSize, 0); if (crcDest!=crcOrig) errorCode = findDiff(mirrorBuffer, dstBuffer, totalTestSize); CHECK (crcDest!=crcOrig, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)", (U32)errorCode, (U32)totalTestSize, dstBuffer[errorCode], mirrorBuffer[errorCode]); }
void rm_digest_update(RmDigest *digest, const unsigned char *data, RmOff size) { switch(digest->type) { case RM_DIGEST_EXT: /* Data is assumed to be a hex representation of a cchecksum. * Needs to be compressed in pure memory first. * * Checksum is not updated but rather overwritten. * */ #define CHAR_TO_NUM(c) (unsigned char)(g_ascii_isdigit(c) ? c - '0' : (c - 'a') + 10) rm_assert_gentle(data); digest->bytes = size / 2; digest->checksum = g_slice_alloc0(digest->bytes); for(unsigned i = 0; i < digest->bytes; ++i) { ((guint8 *)digest->checksum)[i] = (CHAR_TO_NUM(data[2 * i]) << 4) + CHAR_TO_NUM(data[2 * i + 1]); } break; case RM_DIGEST_MD5: case RM_DIGEST_SHA512: case RM_DIGEST_SHA256: case RM_DIGEST_SHA1: g_checksum_update(digest->glib_checksum, (const guchar *)data, size); break; case RM_DIGEST_SPOOKY32: digest->checksum[0].first = spooky_hash32(data, size, digest->checksum[0].first); break; case RM_DIGEST_SPOOKY64: digest->checksum[0].first = spooky_hash64(data, size, digest->checksum[0].first); break; case RM_DIGEST_SPOOKY: spooky_hash128(data, size, (uint64_t *)&digest->checksum[0].first, (uint64_t *)&digest->checksum[0].second); break; case RM_DIGEST_XXHASH: digest->checksum[0].first = XXH64(data, size, digest->checksum[0].first); break; case RM_DIGEST_FARMHASH: digest->checksum[0].first = cfarmhash((const char *)data, size); break; case RM_DIGEST_MURMUR512: case RM_DIGEST_MURMUR256: case RM_DIGEST_MURMUR: for(guint8 block = 0; block < (digest->bytes / 16); block++) { #if RM_PLATFORM_32 MurmurHash3_x86_128(data, size, (uint32_t)digest->checksum[block].first, &digest->checksum[block]); //& #elif RM_PLATFORM_64 MurmurHash3_x64_128(data, size, (uint32_t)digest->checksum[block].first, &digest->checksum[block]); #else #error "Probably not a good idea to compile rmlint on 16bit." #endif } break; case RM_DIGEST_CITY: case RM_DIGEST_CITY256: case RM_DIGEST_CITY512: for(guint8 block = 0; block < (digest->bytes / 16); block++) { /* Opt out for the more optimized version. * This needs the crc command of sse4.2 * (available on Intel Nehalem and up; my amd box doesn't have this though) */ uint128 old = {digest->checksum[block].first, digest->checksum[block].second}; old = CityHash128WithSeed((const char *)data, size, old); memcpy(&digest->checksum[block], &old, sizeof(uint128)); } break; case RM_DIGEST_BASTARD: MurmurHash3_x86_128(data, size, (uint32_t)digest->checksum[0].first, &digest->checksum[0]); uint128 old = {digest->checksum[1].first, digest->checksum[1].second}; old = CityHash128WithSeed((const char *)data, size, old); memcpy(&digest->checksum[1], &old, sizeof(uint128)); break; case RM_DIGEST_CUMULATIVE: { /* This is basically FNV1a, it is just important that the order of * adding data to the hash has no effect on the result, so it can * be used as a lookup key: * * http://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function * */ RmOff hash = 0xcbf29ce484222325; for(gsize i = 0; i < digest->bytes; ++i) { hash ^= ((guint8 *)data)[i % size]; hash *= 0x100000001b3; ((guint8 *)digest->checksum)[i] += hash; } } break; case RM_DIGEST_PARANOID: default: rm_assert_gentle_not_reached(); } }
StageBinary(const uint8_t* data, size_t size) : m_data(data), m_size(size) { m_hash = XXH64(m_data, m_size, 0); }
StageBinary(StageBinaryData data, size_t size) : m_ownedData(std::move(data)), m_data(m_ownedData.get()), m_size(size) { m_hash = XXH64(m_data, m_size, 0); }
int BMK_benchFile(char** fileNamesTable, int nbFiles, int selection) { int fileIdx=0; struct hashFunctionPrototype hashP; U32 hashResult=0; U64 totals = 0; double totalc = 0.; // Init switch (selection) { #ifdef HASH0 case 0 : hashP.hashFunction = HASH0; break; #endif #ifdef HASH1 case 1 : hashP.hashFunction = HASH1; break; #endif #ifdef HASH2 case 2 : hashP.hashFunction = HASH2; break; #endif default: hashP.hashFunction = DEFAULTHASH; } DISPLAY("Selected fn %d", selection); // Loop for each file while (fileIdx<nbFiles) { FILE* inFile; char* inFileName; U64 inFileSize; size_t benchedSize; size_t readSize; char* buffer; char* alignedBuffer; // Check file existence inFileName = fileNamesTable[fileIdx++]; inFile = fopen( inFileName, "rb" ); if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } // Memory allocation & restrictions inFileSize = BMK_GetFileSize(inFileName); benchedSize = (size_t) BMK_findMaxMem(inFileSize); if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; if (benchedSize < inFileSize) { DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); } buffer = (char*)malloc((size_t )benchedSize+16); if(!buffer) { DISPLAY("\nError: not enough memory!\n"); fclose(inFile); return 12; } alignedBuffer = (buffer+15) - (((size_t)(buffer+15)) & 0xF); // align on next 16 bytes boundaries // Fill input buffer DISPLAY("\rLoading %s... \n", inFileName); readSize = fread(alignedBuffer, 1, benchedSize, inFile); fclose(inFile); if(readSize != benchedSize) { DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); free(buffer); return 13; } // Bench XXH32 { int interationNb; double fastestC = 100000000.; DISPLAY("\r%79s\r", ""); // Clean display line for (interationNb = 1; interationNb <= nbIterations; interationNb++) { int nbHashes = 0; int milliTime; DISPLAY("%1i-%-14.14s : %10i ->\r", interationNb, "XXH32", (int)benchedSize); // Hash loop milliTime = BMK_GetMilliStart(); while(BMK_GetMilliStart() == milliTime); milliTime = BMK_GetMilliStart(); while(BMK_GetMilliSpan(milliTime) < TIMELOOP) { int i; for (i=0; i<100; i++) { hashResult = hashP.hashFunction(alignedBuffer, (int)benchedSize, 0); nbHashes++; } } milliTime = BMK_GetMilliSpan(milliTime); if ((double)milliTime < fastestC*nbHashes) fastestC = (double)milliTime/nbHashes; DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, "XXH32", (int)benchedSize, (double)benchedSize / fastestC / 1000.); } DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x%08X\n", "XXH32", (int)benchedSize, (double)benchedSize / fastestC / 1000., hashResult); totals += benchedSize; totalc += fastestC; } // Bench Unaligned XXH32 { int interationNb; double fastestC = 100000000.; DISPLAY("\r%79s\r", ""); // Clean display line for (interationNb = 1; (interationNb <= nbIterations) && ((benchedSize>1)); interationNb++) { int nbHashes = 0; int milliTime; DISPLAY("%1i-%-14.14s : %10i ->\r", interationNb, "(unaligned)", (int)benchedSize); // Hash loop milliTime = BMK_GetMilliStart(); while(BMK_GetMilliStart() == milliTime); milliTime = BMK_GetMilliStart(); while(BMK_GetMilliSpan(milliTime) < TIMELOOP) { int i; for (i=0; i<100; i++) { hashResult = hashP.hashFunction(alignedBuffer+1, (int)benchedSize-1, 0); nbHashes++; } } milliTime = BMK_GetMilliSpan(milliTime); if ((double)milliTime < fastestC*nbHashes) fastestC = (double)milliTime/nbHashes; DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, "XXH32 (unaligned)", (int)(benchedSize-1), (double)(benchedSize-1) / fastestC / 1000.); } DISPLAY("%-16.16s : %10i -> %7.1f MB/s \n", "XXH32 (unaligned)", (int)benchedSize-1, (double)(benchedSize-1) / fastestC / 1000.); } // Bench XXH64 { int interationNb; double fastestC = 100000000.; unsigned long long h64 = 0; DISPLAY("\r%79s\r", ""); // Clean display line for (interationNb = 1; interationNb <= nbIterations; interationNb++) { int nbHashes = 0; int milliTime; DISPLAY("%1i-%-14.14s : %10i ->\r", interationNb, "XXH64", (int)benchedSize); // Hash loop milliTime = BMK_GetMilliStart(); while(BMK_GetMilliStart() == milliTime); milliTime = BMK_GetMilliStart(); while(BMK_GetMilliSpan(milliTime) < TIMELOOP) { int i; for (i=0; i<100; i++) { h64 = XXH64(alignedBuffer, (int)benchedSize, 0); nbHashes++; } } milliTime = BMK_GetMilliSpan(milliTime); if ((double)milliTime < fastestC*nbHashes) fastestC = (double)milliTime/nbHashes; DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, "XXH64", (int)benchedSize, (double)benchedSize / fastestC / 1000.); } DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x%08X%08X\n", "XXH64", (int)benchedSize, (double)benchedSize / fastestC / 1000., (U32)(h64>>32), (U32)(h64)); totals += benchedSize; totalc += fastestC; } free(buffer); }
explicit StageSourceText(std::string_view text) : m_text(text), m_hash(XXH64(m_text.data(), m_text.size(), 0)) {}
// Making a wrapper to fit into the 32 bit api unsigned int XXH64_32(const void* key, unsigned int len, unsigned int seed) { unsigned long long hash = XXH64(key, len, seed); return (unsigned int)(hash & 0xFFFFFFFF); }
static void unitTest(void) { BYTE* testBuff = (BYTE*)malloc(TBSIZE); BYTE* cBuff = (BYTE*)malloc(FSE_COMPRESSBOUND(TBSIZE)); BYTE* verifBuff = (BYTE*)malloc(TBSIZE); size_t errorCode; U32 seed=0, testNb=0, lseed=0; U32 count[256]; if ((!testBuff) || (!cBuff) || (!verifBuff)) { DISPLAY("Not enough memory, exiting ... \n"); free(testBuff); free(cBuff); free(verifBuff); return; } /* FSE_count */ { U32 max, i; for (i=0; i< TBSIZE; i++) testBuff[i] = (FUZ_rand(&lseed) & 63) + '0'; max = '0' + 63; errorCode = FSE_count(count, &max, testBuff, TBSIZE); CHECK(FSE_isError(errorCode), "Error : FSE_count() should have worked"); max -= 1; errorCode = FSE_count(count, &max, testBuff, TBSIZE); CHECK(!FSE_isError(errorCode), "Error : FSE_count() should have failed : value > max"); max = 65000; errorCode = FSE_count(count, &max, testBuff, TBSIZE); CHECK(FSE_isError(errorCode), "Error : FSE_count() should have worked"); } /* FSE_optimalTableLog */ { U32 max, i, tableLog=12; size_t testSize = 999; for (i=0; i< testSize; i++) testBuff[i] = (BYTE)FUZ_rand(&lseed); max = 256; FSE_count(count, &max, testBuff, testSize); tableLog = FSE_optimalTableLog(tableLog, testSize, max); CHECK(tableLog<=8, "Too small tableLog"); } /* FSE_normalizeCount */ { S16 norm[256]; U32 max = 256; FSE_count(count, &max, testBuff, TBSIZE); errorCode = FSE_normalizeCount(norm, 10, count, TBSIZE, max); CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked"); errorCode = FSE_normalizeCount(norm, 8, count, TBSIZE, 256); CHECK(!FSE_isError(errorCode), "Error : FSE_normalizeCount() should have failed (max >= 1<<tableLog)"); /* limit corner case : try to make internal rank overflow */ { U32 i; U32 total = 0; count[0] = 940; count[1] = 910; count[2] = 470; count[3] = 190; count[4] = 90; for(i=5; i<=255; i++) count[i] = 6; for (i=0; i<=255; i++) total += count[i]; errorCode = FSE_normalizeCount(norm, 10, count, total, 255); CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked"); count[0] = 300; count[1] = 300; count[2] = 300; count[3] = 300; count[4] = 50; for(i=5; i<=80; i++) count[i] = 4; total = 0; for (i=0; i<=80; i++) total += count[i]; errorCode = FSE_normalizeCount(norm, 10, count, total, 80); CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked"); } } /* FSE_writeNCount, FSE_readNCount */ { S16 norm[129]; BYTE header[513]; U32 max, tableLog, i; size_t headerSize; for (i=0; i< TBSIZE; i++) testBuff[i] = i % 127; max = 128; errorCode = FSE_count(count, &max, testBuff, TBSIZE); CHECK(FSE_isError(errorCode), "Error : FSE_count() should have worked"); tableLog = FSE_optimalTableLog(0, TBSIZE, max); errorCode = FSE_normalizeCount(norm, tableLog, count, TBSIZE, max); CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked"); headerSize = FSE_NCountWriteBound(max, tableLog); headerSize = FSE_writeNCount(header, 513, norm, max, tableLog); CHECK(FSE_isError(headerSize), "Error : FSE_writeNCount() should have worked"); header[headerSize-1] = 0; errorCode = FSE_writeNCount(header, headerSize-1, norm, max, tableLog); CHECK(!FSE_isError(errorCode), "Error : FSE_writeNCount() should have failed"); CHECK (header[headerSize-1] != 0, "Error : FSE_writeNCount() buffer overwrite"); errorCode = FSE_writeNCount(header, headerSize+1, norm, max, tableLog); CHECK(FSE_isError(errorCode), "Error : FSE_writeNCount() should have worked"); max = 129; errorCode = FSE_readNCount(norm, &max, &tableLog, header, headerSize); CHECK(FSE_isError(errorCode), "Error : FSE_readNCount() should have worked : (error %s)", FSE_getErrorName(errorCode)); max = 64; errorCode = FSE_readNCount(norm, &max, &tableLog, header, headerSize); CHECK(!FSE_isError(errorCode), "Error : FSE_readNCount() should have failed (max too small)"); max = 129; errorCode = FSE_readNCount(norm, &max, &tableLog, header, headerSize-1); CHECK(!FSE_isError(errorCode), "Error : FSE_readNCount() should have failed (size too small)"); { void* smallBuffer = malloc(headerSize-1); /* outbound read can be caught by valgrind */ CHECK(smallBuffer==NULL, "Error : Not enough memory (FSE_readNCount unit test)"); memcpy(smallBuffer, header, headerSize-1); max = 129; errorCode = FSE_readNCount(norm, &max, &tableLog, smallBuffer, headerSize-1); CHECK(!FSE_isError(errorCode), "Error : FSE_readNCount() should have failed (size too small)"); free(smallBuffer); } } /* FSE_buildCTable_raw & FSE_buildDTable_raw */ { U32 ct[FSE_CTABLE_SIZE_U32(8, 256)]; U32 dt[FSE_DTABLE_SIZE_U32(8)]; U64 crcOrig, crcVerif; size_t cSize, verifSize; U32 i; for (i=0; i< TBSIZE; i++) testBuff[i] = (FUZ_rand(&seed) & 63) + '0'; crcOrig = XXH64(testBuff, TBSIZE, 0); errorCode = FSE_buildCTable_raw(ct, 8); CHECK(FSE_isError(errorCode), "FSE_buildCTable_raw should have worked"); errorCode = FSE_buildDTable_raw(dt, 8); CHECK(FSE_isError(errorCode), "FSE_buildDTable_raw should have worked"); cSize = FSE_compress_usingCTable(cBuff, FSE_COMPRESSBOUND(TBSIZE), testBuff, TBSIZE, ct); CHECK(FSE_isError(cSize), "FSE_compress_usingCTable should have worked using raw CTable"); verifSize = FSE_decompress_usingDTable(verifBuff, TBSIZE, cBuff, cSize, dt); CHECK(FSE_isError(verifSize), "FSE_decompress_usingDTable should have worked using raw DTable"); crcVerif = XXH64(verifBuff, verifSize, 0); CHECK(crcOrig != crcVerif, "Raw regenerated data is corrupted"); } /* known corner case */ { BYTE sample8[8] = { 0, 0, 0, 2, 0, 0, 0, 0 }; BYTE* rBuff; errorCode = FSE_compress(cBuff, TBSIZE, sample8, 8); CHECK(FSE_isError(errorCode), "FSE_compress failed compressing sample8"); rBuff = (BYTE*)malloc(errorCode); /* in order to catch read overflow with Valgrind */ CHECK(rBuff==NULL, "Not enough memory for rBuff"); memcpy(rBuff, cBuff, errorCode); errorCode = FSE_decompress(verifBuff, sizeof(sample8), rBuff, errorCode); CHECK(errorCode != sizeof(sample8), "FSE_decompress failed regenerating sample8"); free(rBuff); } free(testBuff); free(cBuff); free(verifBuff); DISPLAY("Unit tests completed\n"); }
std::string HiresTexture::GenBaseName( const u8* texture, size_t texture_size, const u8* tlut, size_t tlut_size, u32 width, u32 height, int format, bool has_mipmaps, bool dump) { std::string name = ""; bool convert = false; HiresTextureCache::iterator convert_iter; if ((!dump || convert) && s_check_native_format) { // try to load the old format first u64 tex_hash = GetHashHiresTexture(texture, (int)texture_size, g_ActiveConfig.iSafeTextureCache_ColorSamples); u64 tlut_hash = 0; if(tlut_size) tlut_hash = GetHashHiresTexture(tlut, (int)tlut_size, g_ActiveConfig.iSafeTextureCache_ColorSamples); name = StringFromFormat("%s_%08x_%i", SConfig::GetInstance().m_strUniqueID.c_str(), (u32)(tex_hash ^ tlut_hash), (u16)format); convert_iter = s_textureMap.find(name); if (convert_iter != s_textureMap.end()) { if (g_ActiveConfig.bConvertHiresTextures) convert = true; else return name; } } if (dump || s_check_new_format || convert) { // checking for min/max on paletted textures u32 min = 0xffff; u32 max = 0; switch (tlut_size) { case 0: break; case 16 * 2: for (size_t i = 0; i < texture_size; i++) { min = std::min<u32>(min, texture[i] & 0xf); min = std::min<u32>(min, texture[i] >> 4); max = std::max<u32>(max, texture[i] & 0xf); max = std::max<u32>(max, texture[i] >> 4); } break; case 256 * 2: for (size_t i = 0; i < texture_size; i++) { min = std::min<u32>(min, texture[i]); max = std::max<u32>(max, texture[i]); } break; case 16384 * 2: for (size_t i = 0; i < texture_size / 2; i++) { min = std::min<u32>(min, Common::swap16(((u16*)texture)[i]) & 0x3fff); max = std::max<u32>(max, Common::swap16(((u16*)texture)[i]) & 0x3fff); } break; } if (tlut_size > 0) { tlut_size = 2 * (max + 1 - min); tlut += 2 * min; } u64 tex_hash = XXH64(texture, texture_size); u64 tlut_hash = 0; if(tlut_size) tlut_hash = XXH64(tlut, tlut_size); std::string basename = s_format_prefix + StringFromFormat("%dx%d%s_%0016" PRIx64, width, height, has_mipmaps ? "_m" : "", tex_hash); std::string tlutname = tlut_size ? StringFromFormat("_%0016" PRIx64, tlut_hash) : ""; std::string formatname = StringFromFormat("_%d", format); std::string fullname = basename + tlutname + formatname; if (convert) { // new texture if (s_textureMap.find(fullname) == s_textureMap.end()) { HiresTextureCacheItem newitem(convert_iter->second.color_map.size()); for (size_t level = 0; level < convert_iter->second.color_map.size(); level++) { std::string newname = fullname; if (level) newname += StringFromFormat("_mip%d", level); newname += convert_iter->second.color_map[level].extension; std::string &src = convert_iter->second.color_map[level].path; size_t postfix = src.find(name); std::string dst = src.substr(0, postfix) + newname; if (File::Rename(src, dst)) { s_check_new_format = true; OSD::AddMessage(StringFromFormat("Rename custom texture %s to %s", src.c_str(), dst.c_str()), 5000); } else { ERROR_LOG(VIDEO, "rename failed"); } newitem.color_map[level] = hires_mip_level(dst, convert_iter->second.color_map[level].extension, convert_iter->second.color_map[level].is_compressed); } s_textureMap.emplace(fullname, newitem); } else { for (size_t level = 0; level < convert_iter->second.color_map.size(); level++) { if (File::Delete(convert_iter->second.color_map[level].path)) { OSD::AddMessage(StringFromFormat("Delete double old custom texture %s", convert_iter->second.color_map[level].path.c_str()), 5000); } else { ERROR_LOG(VIDEO, "delete failed"); } } } s_textureMap.erase(name); } return fullname; }
void Packet::update_signature(uint64_t seed) { signature = XXH64(buffer.start(), buffer.get_data_size(), seed); }
std::string HiresTexture::GenBaseName(const u8* texture, size_t texture_size, const u8* tlut, size_t tlut_size, u32 width, u32 height, int format, bool has_mipmaps, bool dump) { std::string name = ""; bool convert = false; if (!dump && s_check_native_format) { // try to load the old format first u64 tex_hash = GetHashHiresTexture(texture, (int)texture_size, g_ActiveConfig.iSafeTextureCache_ColorSamples); u64 tlut_hash = tlut_size ? GetHashHiresTexture(tlut, (int)tlut_size, g_ActiveConfig.iSafeTextureCache_ColorSamples) : 0; name = StringFromFormat("%s_%08x_%i", SConfig::GetInstance().m_strUniqueID.c_str(), (u32)(tex_hash ^ tlut_hash), (u16)format); if (s_textureMap.find(name) != s_textureMap.end()) { if (g_ActiveConfig.bConvertHiresTextures) convert = true; else return name; } } if (dump || s_check_new_format || convert) { // checking for min/max on paletted textures u32 min = 0xffff; u32 max = 0; switch(tlut_size) { case 0: break; case 16 * 2: for (size_t i = 0; i < texture_size; i++) { min = std::min<u32>(min, texture[i] & 0xf); min = std::min<u32>(min, texture[i] >> 4); max = std::max<u32>(max, texture[i] & 0xf); max = std::max<u32>(max, texture[i] >> 4); } break; case 256 * 2: for (size_t i = 0; i < texture_size; i++) { min = std::min<u32>(min, texture[i]); max = std::max<u32>(max, texture[i]); } break; case 16384 * 2: for (size_t i = 0; i < texture_size/2; i++) { min = std::min<u32>(min, Common::swap16(((u16*)texture)[i]) & 0x3fff); max = std::max<u32>(max, Common::swap16(((u16*)texture)[i]) & 0x3fff); } break; } if (tlut_size > 0) { tlut_size = 2 * (max + 1 - min); tlut += 2 * min; } u64 tex_hash = XXH64(texture, texture_size, 0); u64 tlut_hash = tlut_size ? XXH64(tlut, tlut_size, 0) : 0; std::string basename = s_format_prefix + StringFromFormat("%dx%d%s_%016" PRIx64, width, height, has_mipmaps ? "_m" : "", tex_hash); std::string tlutname = tlut_size ? StringFromFormat("_%016" PRIx64, tlut_hash) : ""; std::string formatname = StringFromFormat("_%d", format); std::string fullname = basename + tlutname + formatname; for (int level = 0; level < 10 && convert; level++) { std::string oldname = name; if (level) oldname += StringFromFormat("_mip%d", level); // skip not existing levels if (s_textureMap.find(oldname) == s_textureMap.end()) continue; for (int i = 0;; i++) { // for hash collisions, padd with an integer std::string newname = fullname; if (level) newname += StringFromFormat("_mip%d", level); if (i) newname += StringFromFormat(".%d", i); // new texture if (s_textureMap.find(newname) == s_textureMap.end()) { std::string src = s_textureMap[oldname]; size_t postfix = src.find_last_of('.'); std::string dst = src.substr(0, postfix - oldname.length()) + newname + src.substr(postfix, src.length() - postfix); if (File::Rename(src, dst)) { s_textureMap.erase(oldname); s_textureMap[newname] = dst; s_check_new_format = true; OSD::AddMessage(StringFromFormat("Rename custom texture %s to %s", oldname.c_str(), newname.c_str()), 5000); } else { ERROR_LOG(VIDEO, "rename failed"); } break; } else { // dst fail already exist, compare content std::string a, b; File::ReadFileToString(s_textureMap[oldname], a); File::ReadFileToString(s_textureMap[newname], b); if (a == b && a != "") { // equal, so remove if (File::Delete(s_textureMap[oldname])) { s_textureMap.erase(oldname); OSD::AddMessage(StringFromFormat("Delete double old custom texture %s", oldname.c_str()), 5000); } else { ERROR_LOG(VIDEO, "delete failed"); } break; } // else continue in this loop with the next higher padding variable } } } // try to match a wildcard template if (!dump && s_textureMap.find(basename + "_*" + formatname) != s_textureMap.end()) return basename + "_*" + formatname; // else generate the complete texture if (dump || s_textureMap.find(fullname) != s_textureMap.end()) return fullname; }