igtl_uint64 igtl_export igtl_us_status_get_crc(igtl_us_status_message * message)
{
igtl_uint64 crc = crc64(0, 0, 0);
crc = crc64((unsigned char*)message, IGTL_US_STATUS_HEADER_SIZE, crc);
return crc;
}
bool ReplBacklog::IsValidCksm(int64 offset, uint64 cksm)
{
uint32 total = m_state->master_repl_offset - offset;
uint64 newcksm = cksm;
if (m_state->repl_backlog_idx >= total)
{
newcksm = crc64(newcksm, (const unsigned char *) (m_repl_backlog + m_state->repl_backlog_idx - total),
total);
}
else
{
newcksm = crc64(newcksm,
(const unsigned char *) (m_repl_backlog + m_state->repl_backlog_size - total
+ m_state->repl_backlog_idx), total - m_state->repl_backlog_idx);
newcksm = crc64(newcksm, (const unsigned char *) m_repl_backlog, m_state->repl_backlog_idx);
}
if (newcksm != m_state->cksm)
{
INFO_LOG(
"Unable to partial resync with the slave for confilct checksum (Slave cksm is: %llu, and server cksm is %llu",
newcksm, m_state->cksm);
return false;
}
return true;
}
igtl_uint64 igtl_export igtl_rts_qtdata_get_crc(igtl_rts_qtdata* rts_qtdata)
{
igtl_uint64 crc;
crc = crc64(0, 0, 0);
crc = crc64((unsigned char*) rts_qtdata, IGTL_RTS_QTDATA_SIZE, crc);
return crc;
}
igtl_uint64 igtl_export igtl_stt_qtdata_get_crc(igtl_stt_qtdata* stt_qtdata)
{
igtl_uint64 crc;
crc = crc64(0, 0, 0);
crc = crc64((unsigned char*) stt_qtdata, IGTL_STT_QTDATA_SIZE, crc);
return crc;
}
igtl_uint64 igtl_export igtl_transform_get_crc(igtl_float32* transform)
{
igtl_uint64 crc = crc64(0, 0, 0);
crc = crc64((unsigned char*)transform, sizeof(igtl_float32)*12, crc);
return crc;
}
igtl_uint64 igtl_export igtl_qtrans_get_crc(igtl_qtrans* pos)
{
igtl_uint64 crc = crc64(0, 0, 0);
crc = crc64((unsigned char*)pos, IGTL_QTRANS_MESSAGE_DEFAULT_SIZE, crc);
return crc;
}
igtl_uint64 igtl_export igtl_capability_get_crc(igtl_capability_info * info, void* capability)
{
igtl_uint64 crc;
igtl_uint64 message_length;
message_length = (igtl_uint32)igtl_capability_get_length(info);
crc = crc64(0, 0, 0);
crc = crc64((unsigned char*) capability, (int)message_length, crc);
return crc;
}
igtl_uint64 igtl_export igtl_ndarray_get_crc(igtl_ndarray_info * info, int type, void* data)
{
igtl_uint64 crc;
igtl_uint64 data_size;
data_size = igtl_ndarray_get_size(info, type);
crc = crc64(0, 0, 0);
crc = crc64((unsigned char*) data, data_size, crc);
return crc;
}
igtl_uint64 igtl_export igtl_polydata_get_crc(igtl_polydata_info * info, int type, void* polydata_message)
{
igtl_uint64 crc;
igtl_uint64 polydata_length;
/*igtl_uint16 i;*/
/*igtl_uint16 nc;*/
polydata_length = (igtl_uint32)igtl_polydata_get_size(info, type);
crc = crc64(0, 0, 0);
crc = crc64((unsigned char*) polydata_message, (int)polydata_length, crc);
return crc;
}
igtl_uint64 igtl_export igtl_image_get_crc(igtl_image_header * header, void* image)
{
igtl_uint64 crc;
igtl_uint64 img_size;
/* calculate image size (we do not call igtl_image_get_data_size()
* because header has already been serialized.
*/
igtl_uint64 si;
igtl_uint64 sj;
igtl_uint64 sk;
igtl_uint64 sp;
if (igtl_is_little_endian())
{
si = BYTE_SWAP_INT16(header->subvol_size[0]);
sj = BYTE_SWAP_INT16(header->subvol_size[1]);
sk = BYTE_SWAP_INT16(header->subvol_size[2]);
}
else
{
si = header->subvol_size[0];
sj = header->subvol_size[1];
sk = header->subvol_size[2];
}
switch (header->scalar_type) {
case IGTL_IMAGE_STYPE_TYPE_INT8:
case IGTL_IMAGE_STYPE_TYPE_UINT8:
sp = 1;
break;
case IGTL_IMAGE_STYPE_TYPE_INT16:
case IGTL_IMAGE_STYPE_TYPE_UINT16:
sp = 2;
break;
case IGTL_IMAGE_STYPE_TYPE_INT32:
case IGTL_IMAGE_STYPE_TYPE_UINT32:
sp = 4;
break;
default:
sp = 0;
break;
}
img_size = si*sj*sk*sp;
crc = crc64(0, 0, 0);
crc = crc64((unsigned char*) header, IGTL_IMAGE_HEADER_SIZE, crc);
crc = crc64((unsigned char*) image, (int)img_size, crc);
return crc;
}
igtl_uint64 igtl_export igtl_qtdata_get_crc(igtl_qtdata_element* qtdatalist, int nitem)
{
igtl_qtdata_element* elem;
int i;
igtl_uint64 crc;
crc = crc64(0, 0, 0);
for (i = 0; i < nitem; i ++)
{
elem = &(qtdatalist[i]);
crc = crc64((unsigned char*) elem, IGTL_QTDATA_ELEMENT_SIZE, crc);
}
return crc;
}
igtl_uint64 igtl_export igtl_point_get_crc(igtl_point_element* pointlist, int nitem)
{
igtl_point_element* elem;
int i;
igtl_uint64 crc;
crc = crc64(0, 0, 0);
for (i = 0; i < nitem; i ++)
{
elem = &(pointlist[i]);
crc = crc64((unsigned char*) elem, IGTL_POINT_ELEMENT_SIZE, crc);
}
return crc;
}
igtl_uint64 igtl_export igtl_command_get_crc(igtl_command_header * header, void* command)
{
igtl_uint64 crc;
igtl_uint32 command_length;
/* convert byte order to get command length */
igtl_command_convert_byte_order(header);
command_length = (igtl_uint32)(header->length);
igtl_command_convert_byte_order(header);
crc = crc64(0, 0, 0);
crc = crc64((unsigned char*) header, IGTL_COMMAND_HEADER_SIZE, crc);
crc = crc64((unsigned char*) command, command_length, crc);
return crc;
}
void ReplBacklog::Feed(Buffer& buffer)
{
const char* p = buffer.GetRawReadBuffer();
size_t len = buffer.ReadableBytes();
m_state->master_repl_offset += len;
/* This is a circular buffer, so write as much data we can at every
* iteration and rewind the "idx" index if we reach the limit. */
while (len)
{
size_t thislen = m_state->repl_backlog_size - m_state->repl_backlog_idx;
if (thislen > len)
thislen = len;
memcpy(m_repl_backlog + m_state->repl_backlog_idx, p, thislen);
m_state->repl_backlog_idx += thislen;
if (m_state->repl_backlog_idx == m_state->repl_backlog_size)
m_state->repl_backlog_idx = 0;
len -= thislen;
p += thislen;
m_state->repl_backlog_histlen += thislen;
}
if (m_state->repl_backlog_histlen > m_state->repl_backlog_size)
m_state->repl_backlog_histlen = m_state->repl_backlog_size;
/* Set the offset of the first byte we have in the backlog. */
m_state->repl_backlog_off = m_state->master_repl_offset - m_state->repl_backlog_histlen + 1;
m_state->cksm = crc64(m_state->cksm, (const unsigned char *) (buffer.GetRawReadBuffer()),
buffer.ReadableBytes());
m_sync_state_change = true;
}
void buf_put (Buf_s **bp)
{
Buf_s *b = *bp;
*bp = NULL;
aver(b->blknum == ((Node_s *)b->d)->blknum);
aver(0 < b->inuse);
++Cache.stat.puts;
--b->inuse;
if (!b->inuse) {
u64 crc = crc64(b->d, BLOCK_SIZE);
if (b->dirty) {
if (crc == b->crc) {
printf("Didn't change %lld\n", b->blknum);
}
//XXX: this can be true. aver(crc != b->crc);
b->crc = crc;
dev_flush(b);
b->dirty = FALSE;
} else {
if (crc != b->crc) {
PRd(b->blknum);
}
aver(crc == b->crc);
}
}
}
igtl_uint64 igtl_export igtl_query_get_crc(igtl_query_header * header, void* query)
{
igtl_uint64 crc;
igtl_uint32 query_length;
/* convert byte order to get query length */
igtl_query_convert_byte_order(header);
query_length = (igtl_uint32)(header->deviceUIDLength);
igtl_query_convert_byte_order(header);
crc = crc64(0, 0, 0);
crc = crc64((unsigned char*) header, IGTL_QUERY_HEADER_SIZE, crc);
crc = crc64((unsigned char*) query, query_length, crc);
return crc;
}
igtl_uint64 igtl_export igtl_string_get_crc(igtl_string_header * header, void* string)
{
igtl_uint64 crc;
igtl_uint64 string_length;
/* convert byte order to get string length */
igtl_string_convert_byte_order(header);
string_length = (igtl_uint32)(header->length);
igtl_string_convert_byte_order(header);
crc = crc64(0, 0, 0);
crc = crc64((unsigned char*) header, IGTL_STRING_HEADER_SIZE, crc);
crc = crc64((unsigned char*) string, (int)string_length, crc);
return crc;
}
igtl_uint64 igtl_export igtl_colortable_get_crc(igtl_colortable_header* header, void* table)
{
igtl_uint64 crc;
igtl_uint64 data_size;
/* calculate header size using igtl_colortable_get_table_size().
* This funciton can be called after byte order conversion, since
* both index and map type field are uint8 and not affected by byte conversion.
*/
data_size = igtl_colortable_get_table_size(header);
crc = crc64(0, 0, 0);
crc = crc64((unsigned char*) header, IGTL_COLORTABLE_HEADER_SIZE, crc);
crc = crc64((unsigned char*) table, (int)data_size, crc);
return crc;
}
igtl_uint64 igtl_US_get_crc(igtl_image_header *header, void* image, igtl_US_tag * tag)
{
igtl_uint64 crc;
crc = igtl_image_get_crc(header, image);
crc = crc64( (unsigned char*) tag, (int) IGTL_US_TAG_SIZE, crc);
return crc;
}
static uint64_t bcache_crc64(struct bcache_super_block *bcs)
{
unsigned char *data = (unsigned char *) bcs;
size_t sz;
data += 8; /* skip csum field */
sz = (unsigned char *) end(bcs) - data;
return crc64(0xFFFFFFFFFFFFFFFFULL, data, sz) ^ 0xFFFFFFFFFFFFFFFFULL;
}
// filedes: file descriptor
static ssize_t rdb_crc_read(int filedes, void *buf, size_t n) {
// ssize_t:signed size_t，in the 32-bit system, ssize_t == int(32bit); in the 64-bit system, ssize_t == long(64bit)
ssize_t len;
len = read(filedes, buf, n);
if (version >= 5) {
checksum = crc64(checksum, buf, len);
}
loaded_bytes += len;
return len;
}
igtl_uint64 igtl_export igtl_bind_get_crc(igtl_bind_info * info, int type, void* bind_message)
{
igtl_uint64 crc;
igtl_uint64 bind_length;
igtl_uint16 i;
igtl_uint16 nc;
bind_length = (igtl_uint64)igtl_bind_get_size(info, type);
crc = crc64(0, 0, 0);
crc = crc64((unsigned char*) bind_message, (int)bind_length, crc);
if (type == IGTL_TYPE_PREFIX_NONE)
{
nc = info->ncmessages;
for (i = 0; i < nc; i ++)
{
crc = crc64((unsigned char*) info->child_info_array[i].ptr, (int)info->child_info_array[i].size, crc);
}
}
return crc;
}
static int l_crc64(lua_State *L)
{
size_t size;
uint64_t crc = 0;
unsigned char outdata[8] = {0x00};
const char *p_str = luaL_checklstring(L, 1, &size);
crc64( (uint8_t*) p_str, size, &crc);
outdata[0] = (uint8_t)(crc >> 56) & 0xff;
outdata[1] = (uint8_t)(crc >> 48) & 0xff;
outdata[2] = (uint8_t)(crc >> 40) & 0xff;
outdata[3] = (uint8_t)(crc >> 32) & 0xff;
outdata[4] = (uint8_t)(crc >> 24) & 0xff;
outdata[5] = (uint8_t)(crc >> 16) & 0xff;
outdata[6] = (uint8_t)(crc >> 8) & 0xff;
outdata[7] = crc & 0xff;
lua_pushlstring(L,(const char *)&outdata, sizeof(outdata));
return 1;
}
uint64_t get_check_value(uint8_t track[], int length, CRC_INFO *crc_info,
CHECK_TYPE check_type) {
uint64_t value;
if (check_type == CHECK_CRC) {
value = crc64(track, length, crc_info);
} else if (check_type == CHECK_CHKSUM) {
value = checksum64(track, length, crc_info);
if (crc_info->length == 16) {
value = value & 0xff;
} else if (crc_info->length == 32) {
value = value & 0xffff;
} else {
msg(MSG_FATAL, "Unsupported checksum length %d\n",crc_info->length);
exit(1);
}
} else if (check_type == CHECK_PARITY) {
value = parity64(track, length, crc_info);
} else {
msg(MSG_FATAL, "Unknown check_type %d\n",check_type);
exit(1);
}
return value;
}
uint64_t crc64(const std::string& str) {
return crc64(str.data(), str.size());
}
void process() {
uint64_t num_errors = 0, num_valid_ops = 0, num_valid_bytes = 0;
entry entry;
int dump_version = processHeader();
/* Exclude the final checksum for RDB >= 5. Will be checked at the end. */
if (dump_version >= 5) {
if (positions[0].size < 8) {
printf("RDB version >= 5 but no room for checksum.\n");
exit(1);
}
positions[0].size -= 8;;
}
level = 1;
while(positions[0].offset < positions[0].size) {
positions[1] = positions[0];
entry = loadEntry();
if (!entry.success) {
uint64_t offset;
int i;
printValid(num_valid_ops, num_valid_bytes);
printErrorStack(&entry);
num_errors++;
num_valid_ops = 0;
num_valid_bytes = 0;
/* search for next valid entry */
offset = positions[0].offset + 1;
i = 0;
while (!entry.success && offset < positions[0].size) {
positions[1].offset = (size_t)offset;
/* find 3 consecutive valid entries */
for (i = 0; i < 3; i++) {
entry = loadEntry();
if (!entry.success) break;
}
/* check if we found 3 consecutive valid entries */
if (i < 3) {
offset++;
}
}
/* print how many bytes we have skipped to find a new valid opcode */
if (offset < positions[0].size) {
printSkipped(offset - positions[0].offset, offset);
}
positions[0].offset = (size_t)offset;
} else {
num_valid_ops++;
num_valid_bytes += positions[1].offset - positions[0].offset;
/* advance position */
positions[0] = positions[1];
}
free(entry.key);
}
/* because there is another potential error,
* print how many valid ops we have processed */
printValid(num_valid_ops, num_valid_bytes);
/* expect an eof */
if (entry.type != REDIS_EOF) {
/* last byte should be EOF, add error */
errors.level = 0;
SHIFT_ERROR(positions[0].offset, "Expected EOF, got %s", types[entry.type]);
/* this is an EOF error so reset type */
entry.type = -1;
printErrorStack(&entry);
num_errors++;
}
/* Verify checksum */
if (dump_version >= 5) {
uint64_t crc = crc64(0,positions[0].data,positions[0].size);
uint64_t crc2;
unsigned char *p = (unsigned char*)positions[0].data+positions[0].size;
crc2 = ((uint64_t)p[0] << 0) |
((uint64_t)p[1] << 8) |
((uint64_t)p[2] << 16) |
((uint64_t)p[3] << 24) |
((uint64_t)p[4] << 32) |
((uint64_t)p[5] << 40) |
((uint64_t)p[6] << 48) |
((uint64_t)p[7] << 56);
if (crc != crc2) {
SHIFT_ERROR(positions[0].offset, "RDB CRC64 does not match.");
} else {
printf("CRC64 checksum is OK\n");
}
}
/* print summary on errors */
if (num_errors) {
printf("\n");
printf("Total unprocessable opcodes: %llu\n",
(unsigned long long) num_errors);
}
}
static void build_mc_points (struct mc_cluster *C) {
if (verbosity >= 3) {
fprintf (stderr, "--> started\n");
}
int N = C->tot_buckets, K = C->points_num, NK = N*K, i, j;
double t1 = get_utime (CLOCK_MONOTONIC);
static struct mc_point_descr Point;
assert (N > 0 && K > 0 && N <= 100000 && K <= 1000);
C->points = malloc (sizeof (mc_point_t) * C->points_num * C->tot_buckets);
assert (C->points);
mc_point_t *ptr = C->points;
for (i = 0; i < N; i++) {
Point.ip = C->buckets[i]->target.s_addr;
Point.port = C->buckets[i]->port;
C->buckets[i]->custom_field = i;
for (j = 0; j < K; j++) {
ptr->target = C->buckets[i];
Point.sugar = j;
ptr->x = crc64 (&Point, sizeof (Point)) * 7047438495421301423LL;
ptr++;
}
}
double t2 = get_utime (CLOCK_MONOTONIC);
sort_points (C->points, NK - 1);
double t3 = get_utime (CLOCK_MONOTONIC);
if (verbosity >= 3) {
long long CC[N];
for (i = 0; i < N; i++) {
CC[i] = 0;
}
for (i = 0; i < NK - 1; i++) {
if (i > NK - 100) {
fprintf (stderr, "%llu %d\n", C->points[i].x, C->points[i].target->custom_field);
}
}
double D = 0, S = ((double)(1LL << 32)) * (double)(1LL << 32), Z = S / N;
for (i = 0; i < NK - 1; i++) {
CC[C->points[i].target->custom_field] += C->points[i+1].x - C->points[i].x;
if ((C->points[i+1].x - C->points[i].x) / Z > 100 || (C->points[i+1].x - C->points[i].x) / Z < 0) {
fprintf (stderr, "%d: %llu %d\n", i, C->points[i].x, C->points[i].target->custom_field);
fprintf (stderr, "%d: %llu %d\n", i + 1, C->points[i + 1].x, C->points[i + 1].target->custom_field);
}
}
CC[C->points[NK - 1].target->custom_field] += C->points[0].x - C->points[NK - 1].x;
long long min = 0x7fffffffffffffffLL, max = 0;
for (i = 0; i < N; i++) {
if (i > N - 100) {
fprintf (stderr, "%.6f\n", CC[i] / Z);
}
if (CC[i] > max) {
max = CC[i];
}
if (CC[i] < min) {
min = CC[i];
}
D += (double) CC[i] * CC[i];
}
double t4 = get_utime (CLOCK_MONOTONIC);
fprintf (stderr, "\nN=%d K=%d avg=%.3f dev=%.3f min=%.3f max=%.3f\n", N, K, 1.0, sqrt (D/(N*Z*Z) - 1.0), min / Z, max / Z);
fprintf (stderr, "Eval time %.3f, sort time %.3f, stat time %.3f\n", t2 - t1, t3 - t2, t4 - t3);
}
for (i = 0; i < N; i++) {
C->buckets[i]->custom_field = 0;
}
}
struct conn_target *calculate_key_target (const char *key, int key_len) {
int i = 0;
unsigned hash;
struct conn_target *S;
static char key_buffer[MAX_KEY_LEN+4];
if (!CC->tot_buckets || key_len <= 0 || key_len > MAX_KEY_LEN) {
return 0;
}
if (key_len >= 3 && key[key_len - 1] == '$' && key[key_len - 2] == '#' && key[key_len - 3] == '@') {
char *tmp;
for (tmp = (char *)(key + key_len - 4); tmp >= (char *)key; --tmp) {
if (*tmp == '#') {
break;
}
}
if (tmp >= (char *)key) {
char *ttt;
unsigned r = strtoul (tmp + 1, &ttt, 10);
if (ttt > tmp + 1 && ttt == key + key_len - 3) {
S = CC->buckets[r % CC->tot_buckets];
return S;
}
}
}
if ((CC->cluster_mode & 255) == CLUSTER_MODE_RANDOM) {
for (i = 0; i < 10; i++) {
S = CC->buckets[lrand48 () % CC->tot_buckets];
if (S && S->active_outbound_connections) {
return S;
}
}
return 0;
}
if ((CC->cluster_mode & 255) == CLUSTER_MODE_FIRSTINT) {
unsigned long long longhash = extract_num (key, key_len, 0);
if (verbosity > 1) {
fprintf (stderr, "extract_num(%.*s) = %llu\n", key_len, key, longhash);
}
if ((long long) longhash == -1) {
return 0;
}
if (CC->step > 0) {
longhash /= CC->step;
}
S = CC->buckets[longhash % CC->tot_buckets];
return S->active_outbound_connections ? S : 0;
}
if ((CC->cluster_mode & 255) > CLUSTER_MODE_FIRSTINT && (CC->cluster_mode & 255) <= CLUSTER_MODE_FIFTHINT) {
int k = (CC->cluster_mode & 255) - (CLUSTER_MODE_FIRSTINT - 1);
char *p1 = (char *) key, *p2;
int clen = key_len;
unsigned long long longhash = 0;
for (i = 0; i < k; i++) {
longhash = extract_num (p1, clen, &p2);
if ((long long) longhash == -1) {
return 0;
}
assert (p2 >= p1 && p2 <= p1 + clen);
clen -= p2 - p1;
p1 = p2;
}
if (CC->step > 0) {
longhash /= CC->step;
}
S = CC->buckets[longhash % CC->tot_buckets];
return S->active_outbound_connections ? S : 0;
}
if (DOT_EXTENSION) {
char *dot_pos = memchr (key, '.', key_len);
if (dot_pos) {
key_len = dot_pos - key;
}
}
if (CC->points) {
unsigned long long x = crc64 (key, key_len);
int a = -1, b = CC->tot_buckets * CC->points_num, c;
while (b - a > 1) {
c = (a + b) >> 1;
if (x < CC->points[c].x) {
b = c;
} else {
a = c;
}
}
assert (CC->points_num > 0);
for (i = 0; i < MAX_RETRIES; i++) {
if (a < 0) {
a += CC->points_num;
}
S = CC->points[a].target;
if (S->active_outbound_connections) {
return S;
}
a--;
}
return 0;
}
/* This function can be installed both in memory and file streams when checksum
* computation is needed. */
void rioGenericUpdateChecksum(rio *r, const void *buf, size_t len) {
r->cksum = crc64(r->cksum, (const unsigned char*)buf,len);
}
/* This function can be installed both in memory and file streams when checksum
* computation is needed. */
void rioGenericUpdateChecksum(rio *r, const void *buf, size_t len) {
r->cksum = crc64(r->cksum,buf,len);
}
