/** Writes a message to the log file.
@param type The type of message to log
@param fileName The filename where this function was called from.
@param lineNum The line number in the file where this function was called from.
@param funcName The name of the function which called this function.
@param msg The message to log
*/
void msg_details(msg_type type, const char *fileName, int lineNum, const char *funcName, const char *msg, ...)
{
msg_init();
/* Construct a string for the user's message */
char msgbuf[1024];
va_list args;
va_start(args, msg);
vsnprintf(msgbuf, 1024, msg, args);
va_end(args);
/* Remove any newlines at the end of the message. */
int msgbufidx = strlen(msgbuf)-1;
while(msgbuf[msgbufidx] == '\n')
{
msgbuf[msgbufidx] = '\0';
msgbufidx--;
}
/* info to prepend to message printed to console */
char typestr[1024];
msg_type_string(type, typestr, 1024);
/* Determine the stream that we are going to print out to: stdout,
* stderr, or don't print to console */
FILE *stream = stdout;
if(type == ERROR || type == FATAL)
stream = stderr;
if(msg_show_type(type) == 0)
stream = NULL;
char timestamp[1024];
msg_timestamp(timestamp, 1024);
char *fileNameCopy = strdup(fileName);
char *shortFileName = basename(fileNameCopy);
/* Print the message to stderr or stdout */
if(stream)
{
msg_start_color(type, stream);
/* Print additional details to console for fatal errors */
if(type == FATAL)
{
fprintf(stream, "%s %s\n", typestr, msgbuf);
fprintf(stream, "%s Occurred at %s:%d in the function %s()\n",
typestr, shortFileName, lineNum, funcName);
}
else
fprintf(stream, "%s %s\n", typestr, msgbuf);
msg_end_color(type, stream);
}
// Not using funcName to try to keep log shorter.
fprintf(f, "%s%s %12s:%-4d %s\n", typestr, timestamp, shortFileName, lineNum, msgbuf);
free(fileNameCopy);
/* Ensure messages are written to the file or console. */
fflush(stream);
fflush(f);
}
static void
req_log(struct msg *req)
{
struct msg *rsp; /* peer message (response) */
int64_t req_time; /* time cost for this request */
char *peer_str; /* peer client ip:port */
uint32_t req_len, rsp_len; /* request and response length */
struct string *req_type; /* request type string */
if (log_loggable(LOG_NOTICE) == 0) {
return;
}
/* a fragment? */
if (req->frag_id != 0 && req->frag_owner != req) {
return;
}
/* conn close normally? */
if (req->mlen == 0) {
return;
}
req_time = nc_usec_now() - req->start_ts;
rsp = req->peer;
req_len = req->mlen;
rsp_len = (rsp != NULL) ? rsp->mlen : 0;
if (req->key_end) {
req->key_end[0] = '\0';
}
/*
* FIXME: add backend addr here
* Maybe we can store addrstr just like server_pool in conn struct
* when connections are resolved
*/
peer_str = nc_unresolve_peer_desc(req->owner->sd);
req_type = msg_type_string(req->type);
log_debug(LOG_NOTICE, "req %"PRIu64" done on c %d req_time %"PRIi64".%03"PRIi64
" msec type %.*s narg %"PRIu32" req_len %"PRIu32" rsp_len %"PRIu32
" key0 '%s' peer '%s' done %d error %d",
req->id, req->owner->sd, req_time / 1000, req_time % 1000,
req_type->len, req_type->data, req->narg, req_len, rsp_len,
req->key_start, peer_str, req->done, req->error);
}
static void
req_log(struct msg *req)
{
struct msg *rsp; /* peer message (response) */
int64_t req_time; /* time cost for this request */
char *peer_str; /* peer client ip:port */
uint32_t req_len, rsp_len; /* request and response length */
struct string *req_type; /* request type string */
struct keypos *kpos;
if (log_loggable(LOG_NOTICE) == 0) {
return;
}
/* a fake request? */
if (req->owner == NULL) {
return;
}
/* a fragment? */
if (req->frag_id != 0 && req->frag_owner != req) {
return;
}
/* conn close normally? */
if (req->mlen == 0) {
return;
}
/*
* there is a race scenario where a requests comes in, the log level is not LOG_NOTICE,
* and before the response arrives you modify the log level to LOG_NOTICE
* using SIGTTIN OR SIGTTOU, then req_log() wouldn't have msg->start_ts set
*/
if (req->start_ts == 0) {
return;
}
req_time = nc_usec_now() - req->start_ts;
rsp = req->peer;
req_len = req->mlen;
rsp_len = (rsp != NULL) ? rsp->mlen : 0;
if (array_n(req->keys) < 1) {
return;
}
kpos = array_get(req->keys, 0);
if (kpos->end != NULL) {
*(kpos->end) = '\0';
}
/*
* FIXME: add backend addr here
* Maybe we can store addrstr just like server_pool in conn struct
* when connections are resolved
*/
peer_str = nc_unresolve_peer_desc(req->owner->sd);
req_type = msg_type_string(req->type);
log_debug(LOG_NOTICE, "req %"PRIu64" done on c %d req_time %"PRIi64".%03"PRIi64
" msec type %.*s narg %"PRIu32" req_len %"PRIu32" rsp_len %"PRIu32
" key0 '%s' peer '%s' done %d error %d",
req->id, req->owner->sd, req_time / 1000, req_time % 1000,
req_type->len, req_type->data, req->narg, req_len, rsp_len,
kpos->start, peer_str, req->done, req->error);
}
rstatus_t
slowlog_command_make_reply(struct context *ctx,
struct conn *conn, struct msg *msg, struct msg *pmsg)
{
rstatus_t status;
uint32_t nkeys;
struct keypos *kp;
char *contents;
uint32_t subcmdlen;
ASSERT(conn->client && !conn->proxy);
ASSERT(msg->request);
ASSERT(pmsg != NULL && !pmsg->request);
ASSERT(msg->owner == conn);
ASSERT(conn->owner == ctx->manager);
nkeys = array_n(msg->keys);
ASSERT(nkeys >= 1);
kp = array_get(msg->keys, 0);
subcmdlen = (uint32_t)(kp->end-kp->start);
if (subcmdlen==strlen("reset")&&!memcmp(kp->start,"reset",subcmdlen)){
if (nkeys != 1) {
goto format_error;
}
slowlog_reset();
status = msg_append_full(pmsg, (uint8_t*)"OK", 2);
if (status != NC_OK) {
conn->err = ENOMEM;
return status;
}
goto done;
} else if (subcmdlen==strlen("id")&&!memcmp(kp->start,"id",subcmdlen)){
int buf_len;
uint8_t buf[30];
long long id;
if (nkeys != 1) {
goto format_error;
}
pthread_rwlock_rdlock(&rwlocker);
id = slowlog_entry_id;
pthread_rwlock_unlock(&rwlocker);
buf_len = nc_scnprintf(buf,30,"%lld",id);
status = msg_append_full(pmsg, buf, (size_t)buf_len);
if (status != NC_OK) {
conn->err = ENOMEM;
return status;
}
goto done;
} else if (subcmdlen==strlen("len")&&!memcmp(kp->start,"len",subcmdlen)){
int len, buf_len;
uint8_t buf[20];
if (nkeys != 1) {
goto format_error;
}
pthread_rwlock_rdlock(&rwlocker);
len = slowlog_len;
pthread_rwlock_unlock(&rwlocker);
buf_len = nc_scnprintf(buf,20,"%d",len);
status = msg_append_full(pmsg, buf, (size_t)buf_len);
if (status != NC_OK) {
conn->err = ENOMEM;
return status;
}
goto done;
} else if (subcmdlen==strlen("get")&&!memcmp(kp->start,"get",subcmdlen)){
int count, sent = 0, buf_len;
uint8_t buf[50];
slowlog_entry *se;
struct string *str;
if (nkeys == 1) {
count = 10;
} else if (nkeys == 2) {
kp = array_get(msg->keys, 1);
count = nc_atoi(kp->start, (kp->end-kp->start));
if (count < 0) {
goto format_error;
}
} else {
goto format_error;
}
pthread_rwlock_rdlock(&rwlocker);
se = STAILQ_FIRST(&slowlog);
while(count-- && se != NULL) {
int nfield;
uint32_t j;
sent++;
buf_len = nc_scnprintf(buf,50,"%d) 1) %lld\r\n",sent, se->id);
status = msg_append_full(pmsg, buf, (size_t)buf_len);
if (status != NC_OK) {
pthread_rwlock_unlock(&rwlocker);
conn->err = ENOMEM;
return status;
}
buf_len = nc_scnprintf(buf,50," 2) %lld\r\n",se->time);
status = msg_append_full(pmsg, buf, (size_t)buf_len);
if (status != NC_OK) {
pthread_rwlock_unlock(&rwlocker);
conn->err = ENOMEM;
return status;
}
buf_len = nc_scnprintf(buf,50," 3) %lld\r\n",se->duration);
status = msg_append_full(pmsg, buf, (size_t)buf_len);
if (status != NC_OK) {
pthread_rwlock_unlock(&rwlocker);
conn->err = ENOMEM;
return status;
}
str = msg_type_string(se->cmdtype);
nfield = 1;
buf_len = nc_scnprintf(buf,50," 4) %d) %s\r\n",nfield++,str->data);
status = msg_append_full(pmsg, buf, (size_t)buf_len);
if (status != NC_OK) {
pthread_rwlock_unlock(&rwlocker);
conn->err = ENOMEM;
return status;
}
buf_len = nc_scnprintf(buf,50," %d) %d\r\n",nfield++,se->keys_count);
status = msg_append_full(pmsg, buf, (size_t)buf_len);
if (status != NC_OK) {
pthread_rwlock_unlock(&rwlocker);
conn->err = ENOMEM;
return status;
}
if (se->keys != NULL) {
for (j = 0; j < array_n(se->keys); j ++) {
str = array_get(se->keys, j);
buf_len = nc_scnprintf(buf,50," %d) ",nfield++);
status = msg_append_full(pmsg, buf, (size_t)buf_len);
if (status != NC_OK) {
pthread_rwlock_unlock(&rwlocker);
conn->err = ENOMEM;
return status;
}
status = msg_append_full(pmsg, str->data, (size_t)str->len);
if (status != NC_OK) {
pthread_rwlock_unlock(&rwlocker);
conn->err = ENOMEM;
return status;
}
status = msg_append_full(pmsg, (uint8_t *)CRLF, CRLF_LEN);
if (status != NC_OK) {
pthread_rwlock_unlock(&rwlocker);
conn->err = ENOMEM;
return status;
}
}
}
se = STAILQ_NEXT(se, next);
}
pthread_rwlock_unlock(&rwlocker);
if (msg_empty(pmsg)) {
status = msg_append_full(pmsg, (uint8_t*)"END", 3);
if (status != NC_OK) {
conn->err = ENOMEM;
return status;
}
goto done;
}
return NC_OK;
} else if (subcmdlen==strlen("overview") &&
!memcmp(kp->start,"overview",subcmdlen)) {
int count, buf_len;
uint8_t buf[50];
int j, idx, id;
struct statistics_oneday *so;
if (nkeys == 1) {
count = 10;
} else if (nkeys == 2) {
kp = array_get(msg->keys, 1);
count = nc_atoi(kp->start, (kp->end-kp->start));
if (count < 0) {
goto format_error;
}
} else {
goto format_error;
}
if (slowlog_statistics == NULL) {
status = msg_append_full(pmsg, (uint8_t*)"END", 3);
if (status != NC_OK) {
conn->err = ENOMEM;
return status;
}
goto done;
}
if (count > statistics_days)
count = statistics_days;
pthread_mutex_lock(&statistics_locker);
idx = today_idx;
id = 1;
while (count--) {
so = &slowlog_statistics[idx];
if (so->year == 0)
break;
buf_len = nc_scnprintf(buf,50,"%d) %d-%d-%d ",id++,so->year+1900,so->mon+1,so->day);
status = msg_append_full(pmsg, buf, (size_t)buf_len);
if (status != NC_OK) {
pthread_mutex_unlock(&statistics_locker);
conn->err = ENOMEM;
return status;
}
for (j = 0; j < statistics_period-1; j ++) {
buf_len = nc_scnprintf(buf,50,"%lld ",so->periods[j]);
status = msg_append_full(pmsg, buf, (size_t)buf_len);
if (status != NC_OK) {
pthread_mutex_unlock(&statistics_locker);
conn->err = ENOMEM;
return status;
}
}
buf_len = nc_scnprintf(buf,50,"%lld\r\n",so->periods[statistics_period-1]);
status = msg_append_full(pmsg, buf, (size_t)buf_len);
if (status != NC_OK) {
pthread_mutex_unlock(&statistics_locker);
conn->err = ENOMEM;
return status;
}
if (--idx < 0) {
idx = statistics_days - 1;
}
}
pthread_mutex_unlock(&statistics_locker);
if (msg_empty(pmsg)) {
status = msg_append_full(pmsg, (uint8_t*)"END", 3);
if (status != NC_OK) {
conn->err = ENOMEM;
return status;
}
goto done;
}
return NC_OK;
} else {
goto format_error;
}
format_error:
contents = "ERR: slowlog command format is error.";
status = msg_append_full(pmsg, (uint8_t *)contents, strlen(contents));
if (status != NC_OK) {
conn->err = ENOMEM;
return status;
}
goto done;
done:
status = msg_append_full(pmsg, (uint8_t *)CRLF, CRLF_LEN);
if (status != NC_OK) {
conn->err = ENOMEM;
return status;
}
return NC_OK;
}
/** Writes a message to the log file.
@param type The type of message to log
@param fileName The filename where this function was called from.
@param lineNum The line number in the file where this function was called from.
@param funcName The name of the function which called this function.
@param msg The message to log
*/
void msg_details(msg_type type, const char *fileName, int lineNum, const char *funcName, const char *msg, ...)
{
msg_init();
/* Construct a string for the user's message */
char msgbuf[1024];
va_list args;
va_start(args, msg);
vsnprintf(msgbuf, 1024, msg, args);
va_end(args);
/* Remove any newlines at the end of the message. */
int msgbufidx = strlen(msgbuf)-1;
while(msgbuf[msgbufidx] == '\n')
{
msgbuf[msgbufidx] = '\0';
msgbufidx--;
}
/* info to prepend to message printed to console */
char typestr[1024];
msg_type_string(type, typestr, 1024);
/* Determine the stream that we are going to print out to: stdout,
* stderr, or don't print to console */
FILE *stream = stdout;
if(type == ERROR || type == FATAL)
stream = stderr;
if(msg_show_type(type) == 0)
stream = NULL;
char timestamp[1024];
msg_timestamp(timestamp, 1024);
char *fileNameCopy = strdup(fileName);
char *shortFileName = basename(fileNameCopy);
/* Print the message to stderr or stdout */
if(stream)
{
// If using a non-standard logfile name, prepend the name to
// the message. This makes it easier to distinguish between
// which process is creating which message if there are
// multiple programs running at once.
char prepend[1024];
if(strcmp(logfile, "log.txt") == 0)
prepend[0] = '\0';
else
snprintf(prepend, 1024, "(%s) ", logfile);
msg_start_color(type, stream);
/* Print additional details to console for fatal errors */
if(type == FATAL)
{
fprintf(stream, "%s %s%s\n", typestr, prepend, msgbuf);
fprintf(stream, "%s %sOccurred at %s:%d in the function %s()\n",
typestr, prepend, shortFileName, lineNum, funcName);
}
else
fprintf(stream, "%s %s%s\n", typestr, prepend, msgbuf);
msg_end_color(type, stream);
}
// Not using funcName to try to keep log shorter.
fprintf(f, "%s%s %12s:%-4d %s\n", typestr, timestamp, shortFileName, lineNum, msgbuf);
free(fileNameCopy);
/* Ensure messages are written to the file or console. */
fflush(stream);
fflush(f);
}
static int check_response(redis_node *rnode, struct msg *r)
{
int ret;
rmtContext *ctx = rnode->ctx;
struct msg *resp, *msg = NULL;
check_data *chdata;
check_unit *cunit;
char extra_err[50];
struct array args;
sds *arg;
struct array *bulks1 = NULL, *bulks2 = NULL;
sds *bulk1, *bulk2;
if (r == NULL) {
return RMT_ERROR;
}
extra_err[0] = '\0';
resp = r->peer;
r->peer = NULL;
array_init(&args, 3, sizeof(sds));
ASSERT(r->request && r->sent);
ASSERT(resp != NULL && resp->request == 0);
cunit = (check_unit *)r->ptr;
chdata = cunit->cdata->data;
if(resp->type == MSG_RSP_REDIS_ERROR){
log_warn("Response from node[%s] for %s is error",
rnode->addr, msg_type_string(r->type));
goto error;
}
if (cunit->state == CHECK_UNIT_STATE_GET_KEY) {
ASSERT(cunit->key == NULL);
ASSERT(cunit->key_type == -1);
ASSERT(cunit->result1 == NULL && cunit->result2 == NULL);
ASSERT(cunit->srnode == rnode);
if (resp->type != MSG_RSP_REDIS_BULK) {
log_error("ERROR: the response type for command 'randomkey' from node[%s] is error: %s",
rnode->addr, msg_type_string(resp->type));
goto error;
}
if (msg_cmp_str(resp, (const uint8_t*)REDIS_REPLY_BULK_NULL,
rmt_strlen(REDIS_REPLY_BULK_NULL)) == 0) {
/* source group may have no keys, stop it */
cunit->cdata->keys_count --;
goto done;
}
cunit->key = redis_msg_response_get_bulk_string(resp);
if (cunit->key == NULL) {
log_error("ERROR: get bulk string from response of node[%s] failed, "
"bulk_len: %"PRIu32", bulk_start: %p",
rnode->addr, resp->bulk_len, resp->bulk_start);
goto error;
}
if (ctx->filter != NULL && !stringmatchlen(ctx->filter, sdslen(ctx->filter),
cunit->key, sdslen(cunit->key), 0)) {
goto done;
}
ASSERT(sdslen(cunit->key) == resp->bulk_len);
msg = msg_get(r->mb, 1, REDIS_DATA_TYPE_CMD);
if (msg == NULL) {
log_error("ERROR: out of memory.");
goto error;
}
arg = array_push(&args);
*arg = sdsnew("type");
arg = array_push(&args);
*arg = sdsdup(cunit->key);
ret = redis_msg_append_command_full_safe(msg, &args);
if (ret != RMT_OK) {
log_error("ERROR: msg append multi bulk len failed.");
goto error;
}
while (array_n(&args) > 0) {
arg = array_pop(&args);
sdsfree(*arg);
}
msg->ptr = cunit;
msg->resp_check = check_response;
ret = prepare_send_msg(rnode, msg, rnode);
if (ret != RMT_OK) {
log_error("ERROR: prepare send msg node[%s] failed.", rnode->addr);
goto error;
}
cunit->state = CHECK_UNIT_STATE_GET_TYPE;
goto next_step;
}
if (cunit->state == CHECK_UNIT_STATE_GET_TYPE) {
ASSERT(cunit->key != NULL);
ASSERT(cunit->key_type == -1);
ASSERT(cunit->result1 == NULL && cunit->result2 == NULL);
ASSERT(cunit->srnode == rnode);
if (resp->type != MSG_RSP_REDIS_STATUS) {
log_error("ERROR: the response type for command 'type' from node[%s] is error: %s",
rnode->addr, msg_type_string(resp->type));
goto error;
}
if (msg_cmp_str(resp, (const uint8_t*)REDIS_REPLY_STATUS_NONE,
rmt_strlen(REDIS_REPLY_STATUS_NONE)) == 0) {
/* This key doesn't exit, may be expired or evicted */
goto done;
}
msg = msg_get(r->mb, 1, REDIS_DATA_TYPE_CMD);
if (msg == NULL) {
log_error("ERROR: out of memory.");
goto error;
}
if (msg_cmp_str(resp, (const uint8_t*)REDIS_REPLY_STATUS_STRING,
rmt_strlen(REDIS_REPLY_STATUS_STRING)) == 0) {
cunit->key_type = REDIS_STRING;
arg = array_push(&args);
*arg = sdsnew("get");
arg = array_push(&args);
*arg = sdsdup(cunit->key);
ret = redis_msg_append_command_full_safe(msg, &args);
if (ret != RMT_OK) {
log_error("ERROR: msg append multi bulk len failed.");
goto error;
}
while (array_n(&args) > 0) {
arg = array_pop(&args);
sdsfree(*arg);
}
} else if (msg_cmp_str(resp, (const uint8_t*)REDIS_REPLY_STATUS_LIST,
rmt_strlen(REDIS_REPLY_STATUS_LIST)) == 0) {
cunit->key_type = REDIS_LIST;
arg = array_push(&args);
*arg = sdsnew("lrange");
arg = array_push(&args);
*arg = sdsdup(cunit->key);
arg = array_push(&args);
*arg = sdsnew("0");
arg = array_push(&args);
*arg = sdsnew("-1");
ret = redis_msg_append_command_full_safe(msg, &args);
if (ret != RMT_OK) {
log_error("ERROR: msg append multi bulk len failed.");
goto error;
}
while (array_n(&args) > 0) {
arg = array_pop(&args);
sdsfree(*arg);
}
} else if (msg_cmp_str(resp, (const uint8_t*)REDIS_REPLY_STATUS_SET,
rmt_strlen(REDIS_REPLY_STATUS_SET)) == 0) {
cunit->key_type = REDIS_SET;
arg = array_push(&args);
*arg = sdsnew("smembers");
arg = array_push(&args);
*arg = sdsdup(cunit->key);
ret = redis_msg_append_command_full_safe(msg, &args);
if (ret != RMT_OK) {
log_error("ERROR: msg append multi bulk len failed.");
goto error;
}
while (array_n(&args) > 0) {
arg = array_pop(&args);
sdsfree(*arg);
}
} else if (msg_cmp_str(resp, (const uint8_t*)REDIS_REPLY_STATUS_ZSET,
rmt_strlen(REDIS_REPLY_STATUS_ZSET)) == 0) {
cunit->key_type = REDIS_ZSET;
arg = array_push(&args);
*arg = sdsnew("zrange");
arg = array_push(&args);
*arg = sdsdup(cunit->key);
arg = array_push(&args);
*arg = sdsnew("0");
arg = array_push(&args);
*arg = sdsnew("-1");
ret = redis_msg_append_command_full_safe(msg, &args);
if (ret != RMT_OK) {
log_error("ERROR: msg append multi bulk len failed.");
goto error;
}
while (array_n(&args) > 0) {
arg = array_pop(&args);
sdsfree(*arg);
}
} else if (msg_cmp_str(resp, (const uint8_t*)REDIS_REPLY_STATUS_HASH,
rmt_strlen(REDIS_REPLY_STATUS_HASH)) == 0) {
cunit->key_type = REDIS_HASH;
arg = array_push(&args);
*arg = sdsnew("hgetall");
arg = array_push(&args);
*arg = sdsdup(cunit->key);
ret = redis_msg_append_command_full_safe(msg, &args);
if (ret != RMT_OK) {
log_error("ERROR: msg append multi bulk len failed.");
goto error;
}
while (array_n(&args) > 0) {
arg = array_pop(&args);
sdsfree(*arg);
}
} else {
log_error("ERROR: response key type from node[%s] is error: ",
rnode->addr);
goto error;
}
msg->ptr = cunit;
msg->resp_check = check_response;
ret = send_msg_to_all(cunit, msg);
if (ret != RMT_OK) {
log_error("ERROR: send msg to source and target group failed.");
goto error;
}
cunit->state = CHECK_UNIT_STATE_GET_VALUE;
goto next_step;
}
if (cunit->state == CHECK_UNIT_STATE_GET_VALUE) {
ASSERT(cunit->key != NULL);
ASSERT(cunit->key_type >= 0);
ASSERT(cunit->result1 == NULL || cunit->result2 == NULL);
if (cunit->key_type == REDIS_STRING) {
if (resp->type != MSG_RSP_REDIS_BULK) {
log_error("ERROR: the response type for %s from node[%s] is error: %s",
rnode->addr, msg_type_string(r->type), msg_type_string(resp->type));
goto error;
}
} else if (cunit->key_type == REDIS_LIST) {
} else if (cunit->key_type == REDIS_SET) {
} else if (cunit->key_type == REDIS_ZSET) {
} else if (cunit->key_type == REDIS_HASH) {
} else {
NOT_REACHED();
}
if (cunit->result1 == NULL) {
cunit->result1 = resp;
resp = NULL;
} else if (cunit->result2 == NULL) {
cunit->result2 = resp;
resp = NULL;
} else {
NOT_REACHED();
}
if (cunit->result1 != NULL && cunit->result2 != NULL) {
if (cunit->key_type == REDIS_SET) {
uint32_t j;
bulks1 = get_multi_bulk_array_from_mbuf_list(cunit->result1->data);
bulks2 = get_multi_bulk_array_from_mbuf_list(cunit->result2->data);
if (bulks1 == NULL || bulks2 == NULL) {
log_error("ERROR: get multi bulk array from mbufs failed");
goto error;
}
if (array_n(bulks1) != array_n(bulks2)) {
chdata->err_inconsistent_value_keys_count ++;
rmt_safe_snprintf(extra_err, 50, ", value is inconsistent\0");
goto error;
}
array_sort(bulks1, string_binary_cmp);
array_sort(bulks2, string_binary_cmp);
for (j = 0; j < array_n(bulks1); j ++) {
bulk1 = array_get(bulks1, j);
bulk2 = array_get(bulks2, j);
if (string_binary_cmp(bulk1, bulk2) != 0) {
chdata->err_inconsistent_value_keys_count ++;
rmt_safe_snprintf(extra_err, 50, ", value is inconsistent\0");
goto error;
}
}
} else if (cunit->key_type == REDIS_HASH) {
struct array *hash_datas1, *hash_datas2;
uint32_t hash_len;
uint32_t j;
struct hash_data *hd1, *hd2;
hash_datas1 = hash_datas2 = NULL;
bulks1 = get_multi_bulk_array_from_mbuf_list(cunit->result1->data);
bulks2 = get_multi_bulk_array_from_mbuf_list(cunit->result2->data);
if (bulks1 == NULL || bulks2 == NULL) {
log_error("ERROR: get multi bulk array from mbufs failed");
goto error;
}
if (array_n(bulks1)%2 != 0 || array_n(bulks2)%2 != 0) {
log_error("ERROR: bad hash value");
goto error;
}
if (array_n(bulks1) != array_n(bulks2)) {
chdata->err_inconsistent_value_keys_count ++;
rmt_safe_snprintf(extra_err, 50, ", value is inconsistent\0");
goto error;
}
hash_len = array_n(bulks1)/2;
hash_datas1 = array_create(hash_len, sizeof(struct hash_data));
hash_datas2 = array_create(hash_len, sizeof(struct hash_data));
for (j = 0; j < hash_len; j ++) {
hd1 = array_push(hash_datas1);
hd2 = array_push(hash_datas2);
bulk1 = array_pop(bulks1);
bulk2 = array_pop(bulks1);
hd1->field = *bulk1;
hd1->value = *bulk2;
bulk1 = array_pop(bulks2);
bulk2 = array_pop(bulks2);
hd2->field = *bulk1;
hd2->value = *bulk2;
}
array_sort(hash_datas1, hash_data_field_cmp);
array_sort(hash_datas2, hash_data_field_cmp);
for (j = 0; j < array_n(bulks1); j ++) {
hd1 = array_get(hash_datas1, j);
hd2 = array_get(hash_datas2, j);
if (string_binary_cmp(hd1->field, hd2->field) != 0) {
chdata->err_inconsistent_value_keys_count ++;
rmt_safe_snprintf(extra_err, 50, ", value is inconsistent\0");
if (hash_datas1 != NULL) {
while (array_n(hash_datas1) > 0) {
hd1 = array_pop(hash_datas1);
sdsfree(hd1->field);
sdsfree(hd1->value);
}
array_destroy(hash_datas1);
hash_datas1 = NULL;
}
if (hash_datas2 != NULL) {
while (array_n(hash_datas2) > 0) {
hd2 = array_pop(hash_datas2);
sdsfree(hd2->field);
sdsfree(hd2->value);
}
array_destroy(hash_datas2);
hash_datas2 = NULL;
}
goto error;
}
if (string_binary_cmp(hd1->value, hd2->value) != 0) {
chdata->err_inconsistent_value_keys_count ++;
rmt_safe_snprintf(extra_err, 50, ", value is inconsistent\0");
if (hash_datas1 != NULL) {
while (array_n(hash_datas1) > 0) {
hd1 = array_pop(hash_datas1);
sdsfree(hd1->field);
sdsfree(hd1->value);
}
array_destroy(hash_datas1);
hash_datas1 = NULL;
}
if (hash_datas2 != NULL) {
while (array_n(hash_datas2) > 0) {
hd2 = array_pop(hash_datas2);
sdsfree(hd2->field);
sdsfree(hd2->value);
}
array_destroy(hash_datas2);
hash_datas2 = NULL;
}
goto error;
}
}
if (hash_datas1 != NULL) {
while (array_n(hash_datas1) > 0) {
hd1 = array_pop(hash_datas1);
sdsfree(hd1->field);
sdsfree(hd1->value);
}
array_destroy(hash_datas1);
hash_datas1 = NULL;
}
if (hash_datas2 != NULL) {
while (array_n(hash_datas2) > 0) {
hd2 = array_pop(hash_datas2);
sdsfree(hd2->field);
sdsfree(hd2->value);
}
array_destroy(hash_datas2);
hash_datas2 = NULL;
}
} else if (msg_data_compare(cunit->result1, cunit->result2) != 0) {
chdata->err_inconsistent_value_keys_count ++;
rmt_safe_snprintf(extra_err, 50, ", value is inconsistent\0");
goto error;
}
msg_put(cunit->result1);
msg_free(cunit->result1);
cunit->result1 = NULL;
msg_put(cunit->result2);
msg_free(cunit->result2);
cunit->result2 = NULL;
if (bulks1 != NULL) {
while (array_n(bulks1) > 0) {
bulk1 = array_pop(bulks1);
sdsfree(*bulk1);
}
array_destroy(bulks1);
bulks1 = NULL;
}
if (bulks2 != NULL) {
while (array_n(bulks2) > 0) {
bulk2 = array_pop(bulks2);
sdsfree(*bulk2);
}
array_destroy(bulks2);
bulks2 = NULL;
}
msg = msg_get(r->mb, 1, REDIS_DATA_TYPE_CMD);
if (msg == NULL) {
log_error("ERROR: out of memory.");
goto error;
}
arg = array_push(&args);
*arg = sdsnew("ttl");
arg = array_push(&args);
*arg = sdsdup(cunit->key);
ret = redis_msg_append_command_full_safe(msg, &args);
if (ret != RMT_OK) {
log_error("ERROR: msg append multi bulk len failed.");
goto error;
}
while (array_n(&args) > 0) {
arg = array_pop(&args);
sdsfree(*arg);
}
msg->ptr = cunit;
msg->resp_check = check_response;
ret = send_msg_to_all(cunit, msg);
if (ret != RMT_OK) {
log_error("ERROR: send msg to source and target group failed.");
goto error;
}
cunit->state = CHECK_UNIT_STATE_GET_EXPIRE;
}
goto next_step;
}
if (cunit->state == CHECK_UNIT_STATE_GET_EXPIRE) {
ASSERT(cunit->key != NULL);
ASSERT(cunit->key_type >= 0);
ASSERT(cunit->result1 == NULL || cunit->result2 == NULL);
if (resp->type != MSG_RSP_REDIS_INTEGER) {
log_error("ERROR: the response type for command 'ttl' from node[%s] is error: %s",
rnode->addr, msg_type_string(resp->type));
goto error;
}
if (cunit->result1 == NULL) {
cunit->result1 = resp;
resp = NULL;
} else if (cunit->result2 == NULL) {
cunit->result2 = resp;
resp = NULL;
} else {
NOT_REACHED();
}
if (cunit->result1 != NULL && cunit->result2 != NULL) {
if (msg_data_compare(cunit->result1, cunit->result2) != 0) {
int mistake = (int)cunit->result1->integer - (int)cunit->result2->integer;
ASSERT(mistake != 0);
if (abs(mistake) > TTL_MISTAKE_CAN_BE_ACCEPT) {
chdata->err_inconsistent_expire_keys_count ++;
rmt_safe_snprintf(extra_err, 50,
", remaining time are %"PRIu32" and %"PRIu32"\0",
cunit->result1->integer, cunit->result2->integer);
goto error;
}
}
/* OK, this key is consistent between source group and target group */
goto done;
}
goto next_step;
}
done:
check_unit_destroy(cunit);
next_step:
msg_put(r);
msg_free(r);
if (resp != NULL) {
msg_put(resp);
msg_free(resp);
}
array_deinit(&args);
return RMT_OK;
error:
chdata->err_check_keys_count ++;
if (cunit->key != NULL) {
log_error("ERROR: key checked failed: %s%s. key(len:%zu, type:%s): %.*s",
get_check_error(cunit), extra_err,
sdslen(cunit->key), get_redis_type_string(cunit->key_type),
sdslen(cunit->key), cunit->key);
} else {
log_error("ERROR: key checked failed: %s%s.",
get_check_error(cunit), extra_err);
}
MSG_DUMP(r, LOG_ERR, 1);
msg_put(r);
msg_free(r);
if (resp != NULL) {
MSG_DUMP(resp, LOG_ERR, 1);
msg_put(resp);
msg_free(resp);
}
if (msg != NULL) {
msg_put(msg);
msg_free(msg);
}
check_unit_destroy(cunit);
while (array_n(&args) > 0) {
arg = array_pop(&args);
sdsfree(*arg);
}
array_deinit(&args);
if (bulks1 != NULL) {
while (array_n(bulks1) > 0) {
bulk1 = array_pop(bulks1);
sdsfree(*bulk1);
}
array_destroy(bulks1);
bulks1 = NULL;
}
if (bulks2 != NULL) {
while (array_n(bulks2) > 0) {
bulk2 = array_pop(bulks2);
sdsfree(*bulk2);
}
array_destroy(bulks2);
bulks2 = NULL;
}
return RMT_OK;
}
