int vbucket_get_vbucket_by_key(VBUCKET_CONFIG_HANDLE vb, const void *key, size_t nkey) {
/* call crc32 directly here it could be changed to some more general
* function when vbucket distribution will support multiple hashing
* algorithms */
uint32_t digest = hash_crc32(key, nkey);
return digest & vb->mask;
}
PyObject* py_strhash(PyObject* self, PyObject *args)
{
const char* msg = NULL;
Py_ssize_t len;
if(!PyArg_ParseTuple(args,"s#",&msg,&len))
return NULL;
int ret=hash_crc32(msg,len);
return Py_BuildValue("I",ret);
}
/* helper function to invoke the correct hash method */
static unsigned int gen_hash(const struct proxy* px, const char* key, unsigned long len)
{
unsigned int hash;
switch (px->lbprm.algo & BE_LB_HASH_FUNC) {
case BE_LB_HFCN_DJB2:
hash = hash_djb2(key, len);
break;
case BE_LB_HFCN_WT6:
hash = hash_wt6(key, len);
break;
case BE_LB_HFCN_CRC32:
hash = hash_crc32(key, len);
break;
case BE_LB_HFCN_SDBM:
/* this is the default hash function */
default:
hash = hash_sdbm(key, len);
break;
}
return hash;
}
/** Common subroutine of pread_bin, pread_compressed and pread_header.
Parameters and return value are the same as for pread_bin,
except the 'header' parameter which is 1 if reading a header, 0 otherwise. */
static int pread_bin_internal(Db *db, off_t pos, char **ret_ptr, int header)
{
struct {
uint32_t chunk_len;
uint32_t crc32;
} info;
couchstore_error_t err = read_skipping_prefixes(db, &pos, sizeof(info), &info);
if (err < 0) {
return err;
}
info.chunk_len = ntohl(info.chunk_len) & ~0x80000000;
if (header) {
if (info.chunk_len < 4 || info.chunk_len > MAX_HEADER_SIZE)
return COUCHSTORE_ERROR_CORRUPT;
info.chunk_len -= 4; //Header len includes CRC len.
}
info.crc32 = ntohl(info.crc32);
char* buf = malloc(info.chunk_len);
if (!buf) {
return COUCHSTORE_ERROR_ALLOC_FAIL;
}
err = read_skipping_prefixes(db, &pos, info.chunk_len, buf);
if (!err && info.crc32 && info.crc32 != hash_crc32(buf, info.chunk_len)) {
err = COUCHSTORE_ERROR_CHECKSUM_FAIL;
}
if (err < 0) {
free(buf);
return err;
}
*ret_ptr = buf;
return info.chunk_len;
}
int
lcbvb_k2vb(lcbvb_CONFIG *cfg, const void *k, lcb_SIZE n)
{
uint32_t digest = hash_crc32(k, n);
return digest % cfg->nvb;
}
/**
* Verifies that we can read all of the data out of the storage tank correctly.
*
* @param check_collection The collection of sums to verify.
* @return Returns true only if all of the objects in the linked list match the expected hash value.
*/
bool_t check_tokyo_tank_verify(inx_t *check_collection) {
stringer_t *data;
check_tank_obj_t *obj;
inx_cursor_t *cursor;
if (!(cursor = inx_cursor_alloc(check_collection))) {
return false;
}
while (status() && (obj = inx_cursor_value_next(cursor))) {
if (!(data = tank_load(TANK_CHECK_DATA_HNUM, obj->tnum, TANK_CHECK_DATA_UNUM, obj->onum))) {
log_info("%lu - tank_get error", obj->onum);
inx_cursor_free(cursor);
return false;
}
if (obj->adler32 != hash_adler32(st_data_get(data), st_length_get(data))) {
log_info("%lu - adler32 error", obj->onum);
inx_cursor_free(cursor);
st_free(data);
return false;
}
if (obj->fletcher32 != hash_fletcher32(st_data_get(data), st_length_get(data))) {
log_info("%lu - fletcher32 error", obj->onum);
inx_cursor_free(cursor);
st_free(data);
return false;
}
if (obj->crc32 != hash_crc32(st_data_get(data), st_length_get(data))) {
log_info("%lu - crc32 error", obj->onum);
inx_cursor_free(cursor);
st_free(data);
return false;
}
if (obj->crc64 != hash_crc64(st_data_get(data), st_length_get(data))) {
log_info("%lu - crc64 error", obj->onum);
inx_cursor_free(cursor);
st_free(data);
return false;
}
if (obj->murmur32 != hash_murmur32(st_data_get(data), st_length_get(data))) {
log_info("%lu - murmur32 error", obj->onum);
inx_cursor_free(cursor);
st_free(data);
return false;
}
if (obj->murmur64 != hash_murmur64(st_data_get(data), st_length_get(data))) {
log_info("%lu - murmur64 error", obj->onum);
inx_cursor_free(cursor);
st_free(data);
return false;
}
st_free(data);
}
inx_cursor_free(cursor);
return true;
}
/**
* Recursively loads the files from a directory and stores them using the tank interface.
*
* @param location The directory path to search for files.
* @return Returns false if an error occurs, otherwise true.
*/
bool_t check_tokyo_tank_load(char *location, inx_t *check_collection, check_tank_opt_t *opts) {
int fd;
multi_t key;
DIR *working;
struct stat info;
check_tank_obj_t *obj;
struct dirent *entry;
char file[1024], *buffer;
if (!(working = opendir(location))) {
log_info("Unable to open the data path. {location = %s}", location);
return false;
}
while (status() && (entry = readdir(working))) {
// Reset.
errno = 0;
bzero(file, 1024);
bzero(&info, sizeof(struct stat));
// Build an absolute path.
snprintf(file, 1024, "%s%s%s", location, "/", entry->d_name);
// If we hit a directory, recursively call the load function.
if (entry->d_type == DT_DIR && *(entry->d_name) != '.') {
if (!check_tokyo_tank_load(file, check_collection, opts)) {
return false;
}
}
// Otherwise if its a regular file try storing it.
else if (entry->d_type == DT_REG && *(entry->d_name) != '.') {
// Read the file.
if ((fd = open(file, O_RDONLY)) < 0) {
log_info("%s - open error", file);
closedir(working);
return false;
}
// How big is the file?
if (fstat(fd, &info) != 0) {
log_info("%s - stat error", file);
closedir(working);
close(fd);
return false;
}
// Allocate a buffer.
if (!(buffer = mm_alloc(info.st_size + 1))) {
log_info("%s - malloc error", file);
closedir(working);
close(fd);
return false;
}
// Clear the buffer.
memset(buffer, 0, info.st_size + 1);
// Read the file.
if (read(fd, buffer, info.st_size) != info.st_size) {
log_info("%s - read error", file);
closedir(working);
mm_free(buffer);
close(fd);
return false;
}
close(fd);
// Data used for verification.
if (!(obj = mm_alloc(sizeof(check_tank_obj_t)))) {
log_info("check_tank allocation failed for the file %s", file);
closedir(working);
mm_free(buffer);
return false;
}
obj->adler32 = hash_adler32(buffer, info.st_size);
obj->fletcher32 = hash_fletcher32(buffer, info.st_size);
obj->crc32 = hash_crc32(buffer, info.st_size);
obj->crc64 = hash_crc64(buffer, info.st_size);
obj->murmur32 = hash_murmur32(buffer, info.st_size);
obj->murmur64 = hash_murmur64(buffer, info.st_size);
// Request the next storage tank.
obj->tnum = tank_cycle();
// Try storing the file data.
if (!(obj->onum = tank_store(TANK_CHECK_DATA_HNUM, obj->tnum, TANK_CHECK_DATA_UNUM, PLACER(buffer, info.st_size), opts->engine))) {
log_info("tank_store failed for the file %s", file);
closedir(working);
mm_free(buffer);
mm_free(obj);
return false;
}
mm_free(buffer);
key = mt_set_type(key, M_TYPE_UINT64);
key.val.u64 = obj->onum;
if (!inx_insert(check_collection, key, obj)) {
log_info("inx_insert failed for the file %s", file);
closedir(working);
mm_free(obj);
return false;
}
}
}
closedir(working);
return true;
}
/*---------------------------------------------------------------------------*
* NAME: update_hash
* DESC: update each hash value.
*---------------------------------------------------------------------------*/
void update_hash(config *conf) {
struct struct_hash *hash;
struct struct_block *block;
unsigned int int_big_endian;
unsigned int int_little_endian;
unsigned int crc = 0;
/* debug */
debug(1, "<-----------------------[enter]\n");
/* verbose */
verbose__("[*] computing the block's hash.\n");
/* for each entry, find the block and fill the size */
for(hash=conf->hash;hash;hash=hash->next) {
for(block=conf->block;block;block=block->next) {
if (block->id == hash->id) {
hash->block = block;
debug(3, "hash (%d) and block[%d] connected\n", block->size, hash->id);
switch(hash->type) {
case HF_BLOCK_CRC32_B : /* CRC32 big-endian 32bits */
/* compute the CRC32 of the block */
crc = hash_crc32(conf->buf_fuzz + block->offset, block->size);
/* convert in big endian */
int_big_endian = htonl(crc);
/* write the hash value */
modify_data(conf, (unsigned char *) &int_big_endian, sizeof(int), hash->offset);
debug(3, "big-endian-32b CRC32 hash: 0x%02x%02x%02x%02x\n",
*(conf->buf_fuzz + hash->offset),
*(conf->buf_fuzz + hash->offset+1),
*(conf->buf_fuzz + hash->offset+2),
*(conf->buf_fuzz + hash->offset+3));
break;
case HF_BLOCK_CRC32_L : /* CRC32 big-endian 32bits */
/* compute the CRC32 of the block */
crc = hash_crc32(conf->buf_fuzz + block->offset, block->size);
/* convert in big endian */
int_big_endian = htonl(crc);
/* convert in little-endian */
memcpy((unsigned char*) &int_little_endian , (unsigned char *) &int_big_endian + 3, sizeof(char));
memcpy((unsigned char*) &int_little_endian + 1, (unsigned char *) &int_big_endian + 2, sizeof(char));
memcpy((unsigned char*) &int_little_endian + 2, (unsigned char *) &int_big_endian + 1, sizeof(char));
memcpy((unsigned char*) &int_little_endian + 3, (unsigned char *) &int_big_endian, sizeof(char));
/* write the hash value */
modify_data(conf, (unsigned char *) &int_little_endian, sizeof(int), hash->offset);
debug(3, "little-endian-32b CRC32 hash: 0x%02x%02x%02x%02x\n",
*(conf->buf_fuzz + hash->offset),
*(conf->buf_fuzz + hash->offset+1),
*(conf->buf_fuzz + hash->offset+2),
*(conf->buf_fuzz + hash->offset+3));
break;
default: /* should never happen */
break;
}
break; /* optimization 8^) */
}
}
}
/* debug */
debug(1, "<-----------------------[quit]\n");
}
