int isSdsRepresentableAsLongLong(sds s, long long *llval) {
return string2ll(s,sdslen(s),llval) ? RR_OK : RR_ERROR;
}
/* Write the append only file buffer on disk.
*
* Since we are required to write the AOF before replying to the client,
* and the only way the client socket can get a write is entering when the
* the event loop, we accumulate all the AOF writes in a memory
* buffer and write it on disk using this function just before entering
* the event loop again.
*
* About the 'force' argument:
*
* When the fsync policy is set to 'everysec' we may delay the flush if there
* is still an fsync() going on in the background thread, since for instance
* on Linux write(2) will be blocked by the background fsync anyway.
* When this happens we remember that there is some aof buffer to be
* flushed ASAP, and will try to do that in the serverCron() function.
*
* However if force is set to 1 we'll write regardless of the background
* fsync. */
void flushAppendOnlyFile(int force) {
ssize_t nwritten;
int sync_in_progress = 0;
if (sdslen(server.aof_buf) == 0) return;
if (server.aof_fsync == AOF_FSYNC_EVERYSEC)
sync_in_progress = bioPendingJobsOfType(REDIS_BIO_AOF_FSYNC) != 0;
if (server.aof_fsync == AOF_FSYNC_EVERYSEC && !force) {
/* With this append fsync policy we do background fsyncing.
* If the fsync is still in progress we can try to delay
* the write for a couple of seconds. */
if (sync_in_progress) {
if (server.aof_flush_postponed_start == 0) {
/* No previous write postponinig, remember that we are
* postponing the flush and return. */
server.aof_flush_postponed_start = server.unixtime;
return;
} else if (server.unixtime - server.aof_flush_postponed_start < 2) {
/* We were already waiting for fsync to finish, but for less
* than two seconds this is still ok. Postpone again. */
return;
}
/* Otherwise fall trough, and go write since we can't wait
* over two seconds. */
server.aof_delayed_fsync++;
redisLog(REDIS_NOTICE,"Asynchronous AOF fsync is taking too long (disk is busy?). Writing the AOF buffer without waiting for fsync to complete, this may slow down Redis.");
}
}
/* If you are following this code path, then we are going to write so
* set reset the postponed flush sentinel to zero. */
server.aof_flush_postponed_start = 0;
/* We want to perform a single write. This should be guaranteed atomic
* at least if the filesystem we are writing is a real physical one.
* While this will save us against the server being killed I don't think
* there is much to do about the whole server stopping for power problems
* or alike */
nwritten = write(server.aof_fd,server.aof_buf,sdslen(server.aof_buf));
if (nwritten != (signed)sdslen(server.aof_buf)) {
/* Ooops, we are in troubles. The best thing to do for now is
* aborting instead of giving the illusion that everything is
* working as expected. */
if (nwritten == -1) {
redisLog(REDIS_WARNING,"Exiting on error writing to the append-only file: %s",strerror(errno));
} else {
redisLog(REDIS_WARNING,"Exiting on short write while writing to "
"the append-only file: %s (nwritten=%ld, "
"expected=%ld)",
strerror(errno),
(long)nwritten,
(long)sdslen(server.aof_buf));
if (ftruncate(server.aof_fd, server.aof_current_size) == -1) {
redisLog(REDIS_WARNING, "Could not remove short write "
"from the append-only file. Redis may refuse "
"to load the AOF the next time it starts. "
"ftruncate: %s", strerror(errno));
}
}
exit(1);
}
server.aof_current_size += nwritten;
/* Re-use AOF buffer when it is small enough. The maximum comes from the
* arena size of 4k minus some overhead (but is otherwise arbitrary). */
if ((sdslen(server.aof_buf)+sdsavail(server.aof_buf)) < 4000) {
sdsclear(server.aof_buf);
} else {
sdsfree(server.aof_buf);
server.aof_buf = sdsempty();
}
/* Don't fsync if no-appendfsync-on-rewrite is set to yes and there are
* children doing I/O in the background. */
if (server.aof_no_fsync_on_rewrite &&
(server.aof_child_pid != -1 || server.rdb_child_pid != -1))
return;
/* Perform the fsync if needed. */
if (server.aof_fsync == AOF_FSYNC_ALWAYS) {
/* aof_fsync is defined as fdatasync() for Linux in order to avoid
* flushing metadata. */
aof_fsync(server.aof_fd); /* Let's try to get this data on the disk */
server.aof_last_fsync = server.unixtime;
} else if ((server.aof_fsync == AOF_FSYNC_EVERYSEC &&
server.unixtime > server.aof_last_fsync)) {
if (!sync_in_progress) aof_background_fsync(server.aof_fd);
server.aof_last_fsync = server.unixtime;
}
}
/* Apply toupper() to every character of the sds string 's'. */
void sdstoupper(sds s) {
int len = sdslen(s), j;
for (j = 0; j < len; j++) s[j] = toupper(s[j]);
}
static int check_response(redis_node *rnode, struct msg *r)
{
int ret;
rmtContext *ctx = rnode->ctx;
struct msg *resp, *msg = NULL;
check_data *chdata;
check_unit *cunit;
char extra_err[50];
struct array args;
sds *arg;
struct array *bulks1 = NULL, *bulks2 = NULL;
sds *bulk1, *bulk2;
if (r == NULL) {
return RMT_ERROR;
}
extra_err[0] = '\0';
resp = r->peer;
r->peer = NULL;
array_init(&args, 3, sizeof(sds));
ASSERT(r->request && r->sent);
ASSERT(resp != NULL && resp->request == 0);
cunit = (check_unit *)r->ptr;
chdata = cunit->cdata->data;
if(resp->type == MSG_RSP_REDIS_ERROR){
log_warn("Response from node[%s] for %s is error",
rnode->addr, msg_type_string(r->type));
goto error;
}
if (cunit->state == CHECK_UNIT_STATE_GET_KEY) {
ASSERT(cunit->key == NULL);
ASSERT(cunit->key_type == -1);
ASSERT(cunit->result1 == NULL && cunit->result2 == NULL);
ASSERT(cunit->srnode == rnode);
if (resp->type != MSG_RSP_REDIS_BULK) {
log_error("ERROR: the response type for command 'randomkey' from node[%s] is error: %s",
rnode->addr, msg_type_string(resp->type));
goto error;
}
if (msg_cmp_str(resp, (const uint8_t*)REDIS_REPLY_BULK_NULL,
rmt_strlen(REDIS_REPLY_BULK_NULL)) == 0) {
/* source group may have no keys, stop it */
cunit->cdata->keys_count --;
goto done;
}
cunit->key = redis_msg_response_get_bulk_string(resp);
if (cunit->key == NULL) {
log_error("ERROR: get bulk string from response of node[%s] failed, "
"bulk_len: %"PRIu32", bulk_start: %p",
rnode->addr, resp->bulk_len, resp->bulk_start);
goto error;
}
if (ctx->filter != NULL && !stringmatchlen(ctx->filter, sdslen(ctx->filter),
cunit->key, sdslen(cunit->key), 0)) {
goto done;
}
ASSERT(sdslen(cunit->key) == resp->bulk_len);
msg = msg_get(r->mb, 1, REDIS_DATA_TYPE_CMD);
if (msg == NULL) {
log_error("ERROR: out of memory.");
goto error;
}
arg = array_push(&args);
*arg = sdsnew("type");
arg = array_push(&args);
*arg = sdsdup(cunit->key);
ret = redis_msg_append_command_full_safe(msg, &args);
if (ret != RMT_OK) {
log_error("ERROR: msg append multi bulk len failed.");
goto error;
}
while (array_n(&args) > 0) {
arg = array_pop(&args);
sdsfree(*arg);
}
msg->ptr = cunit;
msg->resp_check = check_response;
ret = prepare_send_msg(rnode, msg, rnode);
if (ret != RMT_OK) {
log_error("ERROR: prepare send msg node[%s] failed.", rnode->addr);
goto error;
}
cunit->state = CHECK_UNIT_STATE_GET_TYPE;
goto next_step;
}
if (cunit->state == CHECK_UNIT_STATE_GET_TYPE) {
ASSERT(cunit->key != NULL);
ASSERT(cunit->key_type == -1);
ASSERT(cunit->result1 == NULL && cunit->result2 == NULL);
ASSERT(cunit->srnode == rnode);
if (resp->type != MSG_RSP_REDIS_STATUS) {
log_error("ERROR: the response type for command 'type' from node[%s] is error: %s",
rnode->addr, msg_type_string(resp->type));
goto error;
}
if (msg_cmp_str(resp, (const uint8_t*)REDIS_REPLY_STATUS_NONE,
rmt_strlen(REDIS_REPLY_STATUS_NONE)) == 0) {
/* This key doesn't exit, may be expired or evicted */
goto done;
}
msg = msg_get(r->mb, 1, REDIS_DATA_TYPE_CMD);
if (msg == NULL) {
log_error("ERROR: out of memory.");
goto error;
}
if (msg_cmp_str(resp, (const uint8_t*)REDIS_REPLY_STATUS_STRING,
rmt_strlen(REDIS_REPLY_STATUS_STRING)) == 0) {
cunit->key_type = REDIS_STRING;
arg = array_push(&args);
*arg = sdsnew("get");
arg = array_push(&args);
*arg = sdsdup(cunit->key);
ret = redis_msg_append_command_full_safe(msg, &args);
if (ret != RMT_OK) {
log_error("ERROR: msg append multi bulk len failed.");
goto error;
}
while (array_n(&args) > 0) {
arg = array_pop(&args);
sdsfree(*arg);
}
} else if (msg_cmp_str(resp, (const uint8_t*)REDIS_REPLY_STATUS_LIST,
rmt_strlen(REDIS_REPLY_STATUS_LIST)) == 0) {
cunit->key_type = REDIS_LIST;
arg = array_push(&args);
*arg = sdsnew("lrange");
arg = array_push(&args);
*arg = sdsdup(cunit->key);
arg = array_push(&args);
*arg = sdsnew("0");
arg = array_push(&args);
*arg = sdsnew("-1");
ret = redis_msg_append_command_full_safe(msg, &args);
if (ret != RMT_OK) {
log_error("ERROR: msg append multi bulk len failed.");
goto error;
}
while (array_n(&args) > 0) {
arg = array_pop(&args);
sdsfree(*arg);
}
} else if (msg_cmp_str(resp, (const uint8_t*)REDIS_REPLY_STATUS_SET,
rmt_strlen(REDIS_REPLY_STATUS_SET)) == 0) {
cunit->key_type = REDIS_SET;
arg = array_push(&args);
*arg = sdsnew("smembers");
arg = array_push(&args);
*arg = sdsdup(cunit->key);
ret = redis_msg_append_command_full_safe(msg, &args);
if (ret != RMT_OK) {
log_error("ERROR: msg append multi bulk len failed.");
goto error;
}
while (array_n(&args) > 0) {
arg = array_pop(&args);
sdsfree(*arg);
}
} else if (msg_cmp_str(resp, (const uint8_t*)REDIS_REPLY_STATUS_ZSET,
rmt_strlen(REDIS_REPLY_STATUS_ZSET)) == 0) {
cunit->key_type = REDIS_ZSET;
arg = array_push(&args);
*arg = sdsnew("zrange");
arg = array_push(&args);
*arg = sdsdup(cunit->key);
arg = array_push(&args);
*arg = sdsnew("0");
arg = array_push(&args);
*arg = sdsnew("-1");
ret = redis_msg_append_command_full_safe(msg, &args);
if (ret != RMT_OK) {
log_error("ERROR: msg append multi bulk len failed.");
goto error;
}
while (array_n(&args) > 0) {
arg = array_pop(&args);
sdsfree(*arg);
}
} else if (msg_cmp_str(resp, (const uint8_t*)REDIS_REPLY_STATUS_HASH,
rmt_strlen(REDIS_REPLY_STATUS_HASH)) == 0) {
cunit->key_type = REDIS_HASH;
arg = array_push(&args);
*arg = sdsnew("hgetall");
arg = array_push(&args);
*arg = sdsdup(cunit->key);
ret = redis_msg_append_command_full_safe(msg, &args);
if (ret != RMT_OK) {
log_error("ERROR: msg append multi bulk len failed.");
goto error;
}
while (array_n(&args) > 0) {
arg = array_pop(&args);
sdsfree(*arg);
}
} else {
log_error("ERROR: response key type from node[%s] is error: ",
rnode->addr);
goto error;
}
msg->ptr = cunit;
msg->resp_check = check_response;
ret = send_msg_to_all(cunit, msg);
if (ret != RMT_OK) {
log_error("ERROR: send msg to source and target group failed.");
goto error;
}
cunit->state = CHECK_UNIT_STATE_GET_VALUE;
goto next_step;
}
if (cunit->state == CHECK_UNIT_STATE_GET_VALUE) {
ASSERT(cunit->key != NULL);
ASSERT(cunit->key_type >= 0);
ASSERT(cunit->result1 == NULL || cunit->result2 == NULL);
if (cunit->key_type == REDIS_STRING) {
if (resp->type != MSG_RSP_REDIS_BULK) {
log_error("ERROR: the response type for %s from node[%s] is error: %s",
rnode->addr, msg_type_string(r->type), msg_type_string(resp->type));
goto error;
}
} else if (cunit->key_type == REDIS_LIST) {
} else if (cunit->key_type == REDIS_SET) {
} else if (cunit->key_type == REDIS_ZSET) {
} else if (cunit->key_type == REDIS_HASH) {
} else {
NOT_REACHED();
}
if (cunit->result1 == NULL) {
cunit->result1 = resp;
resp = NULL;
} else if (cunit->result2 == NULL) {
cunit->result2 = resp;
resp = NULL;
} else {
NOT_REACHED();
}
if (cunit->result1 != NULL && cunit->result2 != NULL) {
if (cunit->key_type == REDIS_SET) {
uint32_t j;
bulks1 = get_multi_bulk_array_from_mbuf_list(cunit->result1->data);
bulks2 = get_multi_bulk_array_from_mbuf_list(cunit->result2->data);
if (bulks1 == NULL || bulks2 == NULL) {
log_error("ERROR: get multi bulk array from mbufs failed");
goto error;
}
if (array_n(bulks1) != array_n(bulks2)) {
chdata->err_inconsistent_value_keys_count ++;
rmt_safe_snprintf(extra_err, 50, ", value is inconsistent\0");
goto error;
}
array_sort(bulks1, string_binary_cmp);
array_sort(bulks2, string_binary_cmp);
for (j = 0; j < array_n(bulks1); j ++) {
bulk1 = array_get(bulks1, j);
bulk2 = array_get(bulks2, j);
if (string_binary_cmp(bulk1, bulk2) != 0) {
chdata->err_inconsistent_value_keys_count ++;
rmt_safe_snprintf(extra_err, 50, ", value is inconsistent\0");
goto error;
}
}
} else if (cunit->key_type == REDIS_HASH) {
struct array *hash_datas1, *hash_datas2;
uint32_t hash_len;
uint32_t j;
struct hash_data *hd1, *hd2;
hash_datas1 = hash_datas2 = NULL;
bulks1 = get_multi_bulk_array_from_mbuf_list(cunit->result1->data);
bulks2 = get_multi_bulk_array_from_mbuf_list(cunit->result2->data);
if (bulks1 == NULL || bulks2 == NULL) {
log_error("ERROR: get multi bulk array from mbufs failed");
goto error;
}
if (array_n(bulks1)%2 != 0 || array_n(bulks2)%2 != 0) {
log_error("ERROR: bad hash value");
goto error;
}
if (array_n(bulks1) != array_n(bulks2)) {
chdata->err_inconsistent_value_keys_count ++;
rmt_safe_snprintf(extra_err, 50, ", value is inconsistent\0");
goto error;
}
hash_len = array_n(bulks1)/2;
hash_datas1 = array_create(hash_len, sizeof(struct hash_data));
hash_datas2 = array_create(hash_len, sizeof(struct hash_data));
for (j = 0; j < hash_len; j ++) {
hd1 = array_push(hash_datas1);
hd2 = array_push(hash_datas2);
bulk1 = array_pop(bulks1);
bulk2 = array_pop(bulks1);
hd1->field = *bulk1;
hd1->value = *bulk2;
bulk1 = array_pop(bulks2);
bulk2 = array_pop(bulks2);
hd2->field = *bulk1;
hd2->value = *bulk2;
}
array_sort(hash_datas1, hash_data_field_cmp);
array_sort(hash_datas2, hash_data_field_cmp);
for (j = 0; j < array_n(bulks1); j ++) {
hd1 = array_get(hash_datas1, j);
hd2 = array_get(hash_datas2, j);
if (string_binary_cmp(hd1->field, hd2->field) != 0) {
chdata->err_inconsistent_value_keys_count ++;
rmt_safe_snprintf(extra_err, 50, ", value is inconsistent\0");
if (hash_datas1 != NULL) {
while (array_n(hash_datas1) > 0) {
hd1 = array_pop(hash_datas1);
sdsfree(hd1->field);
sdsfree(hd1->value);
}
array_destroy(hash_datas1);
hash_datas1 = NULL;
}
if (hash_datas2 != NULL) {
while (array_n(hash_datas2) > 0) {
hd2 = array_pop(hash_datas2);
sdsfree(hd2->field);
sdsfree(hd2->value);
}
array_destroy(hash_datas2);
hash_datas2 = NULL;
}
goto error;
}
if (string_binary_cmp(hd1->value, hd2->value) != 0) {
chdata->err_inconsistent_value_keys_count ++;
rmt_safe_snprintf(extra_err, 50, ", value is inconsistent\0");
if (hash_datas1 != NULL) {
while (array_n(hash_datas1) > 0) {
hd1 = array_pop(hash_datas1);
sdsfree(hd1->field);
sdsfree(hd1->value);
}
array_destroy(hash_datas1);
hash_datas1 = NULL;
}
if (hash_datas2 != NULL) {
while (array_n(hash_datas2) > 0) {
hd2 = array_pop(hash_datas2);
sdsfree(hd2->field);
sdsfree(hd2->value);
}
array_destroy(hash_datas2);
hash_datas2 = NULL;
}
goto error;
}
}
if (hash_datas1 != NULL) {
while (array_n(hash_datas1) > 0) {
hd1 = array_pop(hash_datas1);
sdsfree(hd1->field);
sdsfree(hd1->value);
}
array_destroy(hash_datas1);
hash_datas1 = NULL;
}
if (hash_datas2 != NULL) {
while (array_n(hash_datas2) > 0) {
hd2 = array_pop(hash_datas2);
sdsfree(hd2->field);
sdsfree(hd2->value);
}
array_destroy(hash_datas2);
hash_datas2 = NULL;
}
} else if (msg_data_compare(cunit->result1, cunit->result2) != 0) {
chdata->err_inconsistent_value_keys_count ++;
rmt_safe_snprintf(extra_err, 50, ", value is inconsistent\0");
goto error;
}
msg_put(cunit->result1);
msg_free(cunit->result1);
cunit->result1 = NULL;
msg_put(cunit->result2);
msg_free(cunit->result2);
cunit->result2 = NULL;
if (bulks1 != NULL) {
while (array_n(bulks1) > 0) {
bulk1 = array_pop(bulks1);
sdsfree(*bulk1);
}
array_destroy(bulks1);
bulks1 = NULL;
}
if (bulks2 != NULL) {
while (array_n(bulks2) > 0) {
bulk2 = array_pop(bulks2);
sdsfree(*bulk2);
}
array_destroy(bulks2);
bulks2 = NULL;
}
msg = msg_get(r->mb, 1, REDIS_DATA_TYPE_CMD);
if (msg == NULL) {
log_error("ERROR: out of memory.");
goto error;
}
arg = array_push(&args);
*arg = sdsnew("ttl");
arg = array_push(&args);
*arg = sdsdup(cunit->key);
ret = redis_msg_append_command_full_safe(msg, &args);
if (ret != RMT_OK) {
log_error("ERROR: msg append multi bulk len failed.");
goto error;
}
while (array_n(&args) > 0) {
arg = array_pop(&args);
sdsfree(*arg);
}
msg->ptr = cunit;
msg->resp_check = check_response;
ret = send_msg_to_all(cunit, msg);
if (ret != RMT_OK) {
log_error("ERROR: send msg to source and target group failed.");
goto error;
}
cunit->state = CHECK_UNIT_STATE_GET_EXPIRE;
}
goto next_step;
}
if (cunit->state == CHECK_UNIT_STATE_GET_EXPIRE) {
ASSERT(cunit->key != NULL);
ASSERT(cunit->key_type >= 0);
ASSERT(cunit->result1 == NULL || cunit->result2 == NULL);
if (resp->type != MSG_RSP_REDIS_INTEGER) {
log_error("ERROR: the response type for command 'ttl' from node[%s] is error: %s",
rnode->addr, msg_type_string(resp->type));
goto error;
}
if (cunit->result1 == NULL) {
cunit->result1 = resp;
resp = NULL;
} else if (cunit->result2 == NULL) {
cunit->result2 = resp;
resp = NULL;
} else {
NOT_REACHED();
}
if (cunit->result1 != NULL && cunit->result2 != NULL) {
if (msg_data_compare(cunit->result1, cunit->result2) != 0) {
int mistake = (int)cunit->result1->integer - (int)cunit->result2->integer;
ASSERT(mistake != 0);
if (abs(mistake) > TTL_MISTAKE_CAN_BE_ACCEPT) {
chdata->err_inconsistent_expire_keys_count ++;
rmt_safe_snprintf(extra_err, 50,
", remaining time are %"PRIu32" and %"PRIu32"\0",
cunit->result1->integer, cunit->result2->integer);
goto error;
}
}
/* OK, this key is consistent between source group and target group */
goto done;
}
goto next_step;
}
done:
check_unit_destroy(cunit);
next_step:
msg_put(r);
msg_free(r);
if (resp != NULL) {
msg_put(resp);
msg_free(resp);
}
array_deinit(&args);
return RMT_OK;
error:
chdata->err_check_keys_count ++;
if (cunit->key != NULL) {
log_error("ERROR: key checked failed: %s%s. key(len:%zu, type:%s): %.*s",
get_check_error(cunit), extra_err,
sdslen(cunit->key), get_redis_type_string(cunit->key_type),
sdslen(cunit->key), cunit->key);
} else {
log_error("ERROR: key checked failed: %s%s.",
get_check_error(cunit), extra_err);
}
MSG_DUMP(r, LOG_ERR, 1);
msg_put(r);
msg_free(r);
if (resp != NULL) {
MSG_DUMP(resp, LOG_ERR, 1);
msg_put(resp);
msg_free(resp);
}
if (msg != NULL) {
msg_put(msg);
msg_free(msg);
}
check_unit_destroy(cunit);
while (array_n(&args) > 0) {
arg = array_pop(&args);
sdsfree(*arg);
}
array_deinit(&args);
if (bulks1 != NULL) {
while (array_n(bulks1) > 0) {
bulk1 = array_pop(bulks1);
sdsfree(*bulk1);
}
array_destroy(bulks1);
bulks1 = NULL;
}
if (bulks2 != NULL) {
while (array_n(bulks2) > 0) {
bulk2 = array_pop(bulks2);
sdsfree(*bulk2);
}
array_destroy(bulks2);
bulks2 = NULL;
}
return RMT_OK;
}
sds sdscatsds(sds s, const sds t) {
return sdscatlen(s, t, sdslen(t));
}
void mixObjectDigest(unsigned char *digest, robj *o) {
o = getDecodedObject(o);
mixDigest(digest,o->ptr,sdslen(o->ptr));
decrRefCount(o);
}
void redis_check_data(rmtContext *ctx, int type)
{
int i;
int threads_count;
long long keys_count = 0, keys_count_per_thread;
long long keys_count_left, keys_count_threads_hold;
thread_data **threads = NULL;
long long starttime, endtime;
long long checked_keys_count = 0;
long long err_check_keys_count = 0;
long long err_others_keys_count = 0;
long long err_inconsistent_value_keys_count = 0;
long long err_inconsistent_expire_keys_count = 0;
RMT_NOTUSED(type);
if (ctx == NULL || ctx->source_addr == NULL) {
goto error;
}
if (ctx->source_type == GROUP_TYPE_RDBFILE ||
ctx->target_type == GROUP_TYPE_RDBFILE) {
log_error("ERROR: source and target type can not be rdb file for redis_check command");
goto error;
}
signal(SIGPIPE, SIG_IGN);
if (array_n(&ctx->args) == 1) {
sds *str = array_get(&ctx->args, 0);
keys_count = rmt_atoll(*str,sdslen(*str));
}
if (keys_count <= 0) {
keys_count = 1000;
}
keys_count_threads_hold = 0;
keys_count_left = keys_count;
if (keys_count <= 0) {
log_error("ERROR: keys count is less than 0");
goto error;
}
threads_count = ctx->thread_count;
if (threads_count <= 0) {
log_error("ERROR: thread count is less than 0.");
goto error;
}
if (keys_count < threads_count) {
threads_count = (int)keys_count;
}
if (keys_count <= 100) {
threads_count = 1;
}
ctx->thread_count = threads_count;
threads = rmt_zalloc(threads_count * sizeof(*threads));
if (threads == NULL) {
log_error("Error: out of memory.");
goto error;
}
keys_count_per_thread = keys_count/threads_count;
//Create the thread data
for(i = 0; i < threads_count; i ++){
threads[i] = check_thread_data_create(ctx);
if (threads[i] == NULL) {
log_error("ERROR: create check thread %d failed.", i);
goto error;
}
threads[i]->id = i;
threads[i]->keys_count = keys_count_per_thread;
keys_count_left -= threads[i]->keys_count;
}
i = 0;
while (keys_count_left > 0) {
threads[i]->keys_count ++;
keys_count_left --;
i ++;
if(i >= threads_count) i = 0;
}
for (i = 0; i < threads_count; i ++) {
keys_count_threads_hold += threads[i]->keys_count;
}
if(keys_count_threads_hold != keys_count) {
log_error("ERROR: keys_count_threads_hold %lld != keys_count %lld",
keys_count_threads_hold, keys_count);
goto error;
}
log_stdout("Check job is running...");
starttime = rmt_msec_now();
//Run the job
for(i = 0; i < threads_count; i ++){
pthread_create(&threads[i]->thread_id,
NULL, check_thread_run, threads[i]);
}
//Wait for the job finish
for(i = 0; i < threads_count; i ++){
pthread_join(threads[i]->thread_id, NULL);
}
endtime = rmt_msec_now();
for(i = 0; i < threads_count; i ++){
checked_keys_count +=
threads[i]->finished_keys_count;
check_data *chdata = threads[i]->data;
err_check_keys_count +=
chdata->err_check_keys_count;
err_inconsistent_value_keys_count +=
chdata->err_inconsistent_value_keys_count;
err_inconsistent_expire_keys_count +=
chdata->err_inconsistent_expire_keys_count;
}
err_others_keys_count = err_check_keys_count -
err_inconsistent_value_keys_count -
err_inconsistent_expire_keys_count;
/* Show the check result */
log_stdout("");
log_stdout("Checked keys: %lld", checked_keys_count);
log_stdout("\033[%dmInconsistent value keys: %lld\033[0m",
err_inconsistent_value_keys_count == 0?0:31,
err_inconsistent_value_keys_count);
log_stdout("\033[%dmInconsistent expire keys : %lld\033[0m",
err_inconsistent_expire_keys_count == 0?0:33,
err_inconsistent_expire_keys_count);
log_stdout("\033[%dmOther check error keys: %lld\033[0m",
err_others_keys_count == 0?0:31,
err_others_keys_count);
log_stdout("Checked OK keys: %lld",
checked_keys_count - err_check_keys_count);
log_stdout("");
if (err_check_keys_count == 0) {
log_stdout("\033[32mAll keys checked OK!\033[0m");
}
log_stdout("Check job finished, used %.3fs",
(float)(endtime-starttime)/1000);
error:
if (threads != NULL) {
for(i = 0; i < threads_count; i ++){
check_thread_data_destroy(threads[i]);
}
rmt_free(threads);
}
}
static int se_try_read(lua_State *L, int fd, int size, sds *pcache)
{
char sbuf[4 << 10];
char *cache = *pcache;
char *buf;
int bufsize;
int nread;
if (cache) {
bufsize = sdsavail(cache);
buf = cache + sdslen(cache);
printf("continue try read: %d / %d\n", bufsize, size);
} else { // first try
bufsize = size > 0 ? size : size < 0 ? -size : sizeof(sbuf);
if (bufsize <= sizeof(sbuf)) {
buf = sbuf;
} else {
cache = sdsnewlen(NULL, bufsize);
oom_check(cache);
sdsclear(cache);
*pcache = cache;
buf = cache;
}
printf("try read: %d / %d\n", bufsize, size);
}
nread = read(fd, buf, bufsize);
if (nread > 0) {
if (size <= 0 || nread == bufsize) { // done
if (cache) {
lua_pushlstring(L, cache, sdslen(cache) + nread);
sdsfree(cache);
*pcache = NULL;
} else {
lua_pushlstring(L, buf, nread);
}
printf("read done: %d / %d / %d\n", nread, bufsize, size);
return 1;
}
// partial read
if (!cache) {
cache = sdsnewlen(NULL, bufsize);
oom_check(cache);
sdsclear(cache);
*pcache = cache;
memcpy(cache, buf, nread);
}
sdsIncrLen(cache, nread);
printf("partial read: %d / %d / %d\n", nread, bufsize, size);
return -1;
}
if (nread == 0)
return se_read_error(L, pcache, "EOF");
if (errno == EAGAIN || errno == EWOULDBLOCK)
return -1;
se_assert(L, errno != EBADF, "read(%d) error", fd);
return se_read_error(L, pcache, strerror(errno));
}
static int __init testredis_init(void)
{
int fd;
int fails = 0;
redisReply *reply;
printk(KERN_INFO "testredis_init() called\n");
reply = redisConnect(&fd, SERVER_IP, SERVER_PORT);
if (reply != NULL) {
printk(KERN_INFO "Connection error: %s", reply->reply);
return 1;
}
/* test 1 */
printk(KERN_INFO "\n#1 Is able to deliver commands: ");
reply = redisCommand(fd, "PING");
test_cond(reply->type == REDIS_REPLY_STRING &&
strcasecmp(reply->reply, "pong") == 0)
/* Switch to DB 9 for testing, now that we know we can chat. */
reply = redisCommand(fd, "SELECT 9");
freeReplyObject(reply);
/* Make sure the DB is emtpy */
reply = redisCommand(fd, "DBSIZE");
if (reply->type != REDIS_REPLY_INTEGER || reply->integer != 0) {
printk(KERN_INFO
"Sorry DB 9 is not empty, test can not continue\n");
return 1;
} else {
printk(KERN_INFO "DB 9 is empty... test can continue\n");
}
freeReplyObject(reply);
/* test 2 */
printk(KERN_INFO "#2 Is a able to send commands verbatim: ");
reply = redisCommand(fd, "SET foo bar");
test_cond(reply->type == REDIS_REPLY_STRING &&
strcasecmp(reply->reply, "ok") == 0) freeReplyObject(reply);
/* test 3 */
printk(KERN_INFO "#3 %%s String interpolation works: ");
reply = redisCommand(fd, "SET %s %s", "foo", "hello world");
freeReplyObject(reply);
reply = redisCommand(fd, "GET foo");
test_cond(reply->type == REDIS_REPLY_STRING &&
strcmp(reply->reply, "hello world") == 0);
freeReplyObject(reply);
/* test 4 & 5 */
printk(KERN_INFO "#4 %%b String interpolation works: ");
reply = redisCommand(fd, "SET %b %b", "foo", 3, "hello\x00world", 11);
freeReplyObject(reply);
reply = redisCommand(fd, "GET foo");
test_cond(reply->type == REDIS_REPLY_STRING &&
memcmp(reply->reply, "hello\x00world", 11) == 0)
printk(KERN_INFO "#5 binary reply length is correct: ");
test_cond(sdslen(reply->reply) == 11) freeReplyObject(reply);
/* test 6 */
printk(KERN_INFO "#6 can parse nil replies: ");
reply = redisCommand(fd,"GET nokey");
printk(KERN_INFO "Received %c %d\n", reply->type, reply->type);
test_cond(reply->type == REDIS_REPLY_NIL) freeReplyObject(reply);
/* test 7 */
printk(KERN_INFO "#7 can parse integer replies: ");
reply = redisCommand(fd, "INCR mycounter");
test_cond(reply->type == REDIS_REPLY_INTEGER
&& reply->integer == 1) freeReplyObject(reply);
/* test 8 */
printk(KERN_INFO "#8 can parse multi bulk replies: ");
freeReplyObject(redisCommand(fd, "LPUSH mylist foo"));
freeReplyObject(redisCommand(fd, "LPUSH mylist bar"));
reply = redisCommand(fd, "LRANGE mylist 0 -1");
test_cond(reply->type == REDIS_REPLY_ARRAY &&
reply->elements == 2 &&
!memcmp(reply->element[0]->reply, "bar", 3) &&
!memcmp(reply->element[1]->reply, "foo", 3))
freeReplyObject(reply);
/* Clean DB 9 */
reply = redisCommand(fd, "FLUSHDB");
freeReplyObject(reply);
if (fails == 0) {
printk(KERN_INFO "ALL TESTS PASSED\n");
} else {
printk(KERN_INFO "*** %d TESTS FAILED ***\n", fails);
}
return 0;
}
static int
initFromConfigBuffer (sds config, conf_t * conf)
{
char *err = NULL;
int linenum = 0, totlines, i;
sds *lines;
lines = sdssplitlen (config, sdslen (config), "\n", 1, &totlines);
for (i = 0; i < totlines; i++)
{
sds *argv;
int argc;
linenum = i + 1;
lines[i] = sdstrim (lines[i], " \t\r\n");
/* Skip comments and blank lines */
if (lines[i][0] == '#' || lines[i][0] == '\0')
continue;
/* Split into arguments */
argv = sdssplitargs (lines[i], &argc);
if (argv == NULL)
{
err = "Unbalanced quotes in configuration line";
goto err;
}
/* Skip this line if the resulting command vector is empty. */
if (argc == 0)
{
sdsfreesplitres (argv, argc);
continue;
}
sdstolower (argv[0]);
if (!strcasecmp (argv[0], "zookeeper") && argc == 2)
{
conf->zk_addr = strdup (argv[1]);
}
else if (!strcasecmp (argv[0], "cluster_name") && argc == 2)
{
conf->cluster_name = strdup (argv[1]);
}
else if (!strcasecmp (argv[0], "port") && argc == 2)
{
conf->port = atoi (argv[1]);
if (conf->port < 0 || conf->port > 65535)
{
sdsfreesplitres (argv, argc);
err = "Invalid port";
goto err;
}
}
else if (!strcasecmp (argv[0], "daemonize") && argc == 2)
{
if (!strcasecmp (argv[1], "yes"))
{
conf->daemonize = 1;
}
else if (!strcasecmp (argv[1], "no"))
{
conf->daemonize = 0;
}
else
{
sdsfreesplitres (argv, argc);
err = "argument must be 'yes' or 'no'";
goto err;
}
}
else if (!strcasecmp (argv[0], "num_conn_per_gw") && argc == 2)
{
conf->capi_conf.num_conn_per_gw = atoi (argv[1]);
if (conf->capi_conf.num_conn_per_gw < 1
|| conf->capi_conf.num_conn_per_gw > 32)
{
sdsfreesplitres (argv, argc);
err = "Invalid num_conn_per_gw value";
goto err;
}
}
else if (!strcasecmp (argv[0], "init_timeout_millis") && argc == 2)
{
conf->capi_conf.init_timeout_millis = atoi (argv[1]);
if (conf->capi_conf.init_timeout_millis < 3000
|| conf->capi_conf.init_timeout_millis > 600000)
{
sdsfreesplitres (argv, argc);
err = "Invalid init_timeout_millis value";
goto err;
}
}
else if (!strcasecmp (argv[0], "log_level") && argc == 2)
{
if (!strcasecmp (argv[1], "NOLOG"))
{
conf->capi_conf.log_level = ARC_LOG_LEVEL_NOLOG;
}
else if (!strcasecmp (argv[1], "ERROR"))
{
conf->capi_conf.log_level = ARC_LOG_LEVEL_ERROR;
}
else if (!strcasecmp (argv[1], "WARN"))
{
conf->capi_conf.log_level = ARC_LOG_LEVEL_WARN;
}
else if (!strcasecmp (argv[1], "INFO"))
{
conf->capi_conf.log_level = ARC_LOG_LEVEL_INFO;
}
else if (!strcasecmp (argv[1], "DEBUG"))
{
conf->capi_conf.log_level = ARC_LOG_LEVEL_DEBUG;
}
else
{
sdsfreesplitres (argv, argc);
err = "Invalid log_level value";
goto err;
}
}
else if (!strcasecmp (argv[0], "log_file_prefix") && argc == 2)
{
if (argv[1][0] != '\0')
{
conf->capi_conf.log_file_prefix = strdup (argv[1]);
}
}
else if (!strcasecmp (argv[0], "max_fd") && argc == 2)
{
conf->capi_conf.max_fd = atoi (argv[1]);
if (conf->capi_conf.max_fd < 1024 || conf->capi_conf.max_fd > 16000)
{
sdsfreesplitres (argv, argc);
err = "Invalid max_fd value";
goto err;
}
}
else if (!strcasecmp (argv[0], "conn_reconnect_millis") && argc == 2)
{
conf->capi_conf.conn_reconnect_millis = atoi (argv[1]);
if (conf->capi_conf.conn_reconnect_millis < 100
|| conf->capi_conf.conn_reconnect_millis > 600000)
{
sdsfreesplitres (argv, argc);
err = "Invalid conn_reconnect_millis value";
goto err;
}
}
else if (!strcasecmp (argv[0], "zk_reconnect_millis") && argc == 2)
{
conf->capi_conf.zk_reconnect_millis = atoi (argv[1]);
if (conf->capi_conf.zk_reconnect_millis < 100
|| conf->capi_conf.zk_reconnect_millis > 600000)
{
sdsfreesplitres (argv, argc);
err = "Invalid zk_reconnect_millis value";
goto err;
}
}
else if (!strcasecmp (argv[0], "zk_session_timeout_millis")
&& argc == 2)
{
conf->capi_conf.zk_session_timeout_millis = atoi (argv[1]);
if (conf->capi_conf.zk_session_timeout_millis < 1000
|| conf->capi_conf.zk_session_timeout_millis > 600000)
{
sdsfreesplitres (argv, argc);
err = "Invalid zk_session_timeout_millis value";
goto err;
}
}
else if (!strcasecmp (argv[0], "local_proxy_query_timeout_millis")
&& argc == 2)
{
conf->query_timeout_millis = atoi (argv[1]);
if (conf->query_timeout_millis < 1000
|| conf->query_timeout_millis > 600000)
{
sdsfreesplitres (argv, argc);
err = "Invalid local_proxy_query_timeout_millis";
goto err;
}
}
else
{
sdsfreesplitres (argv, argc);
err = "Bad directive or wrong number of arguments";
goto err;
}
sdsfreesplitres (argv, argc);
}
sdsfreesplitres (lines, totlines);
return 0;
err:
fprintf (stderr, "\n*** FATAL CONFIG FILE ERROR ***\n");
fprintf (stderr, "Reading the configuration file, at line %d\n", linenum);
fprintf (stderr, ">>> '%s'\n", lines[i]);
fprintf (stderr, "%s\n", err);
sdsfreesplitres (lines, totlines);
return -1;
}
static int
processMultiBulk (client_t * c)
{
char *newline = NULL;
int pos = 0, ok;
long long ll;
if (c->multibulklen == 0)
{
newline = strchr (c->querybuf, '\r');
if (newline == NULL)
{
if (sdslen (c->querybuf) > REDIS_INLINE_MAX_SIZE)
{
addReplyStr (c,
"-ERR Protocol error: too big mbulk count string\r\n");
c->flags |= REDIS_CLOSE_AFTER_REPLY;
}
return -1;
}
if (newline - c->querybuf > sdslen (c->querybuf) - 2)
return -1;
ok = string2ll (c->querybuf + 1, newline - (c->querybuf + 1), &ll);
if (!ok || ll > 1024 * 1024)
{
addReplyStr (c,
"-ERR Protocol error: invalid multibulk length\r\n");
c->flags |= REDIS_CLOSE_AFTER_REPLY;
return -1;
}
c->multibulklen = ll;
c->argv = malloc (sizeof (sds) * c->multibulklen);
c->argvlen = malloc (sizeof (size_t) * c->multibulklen);
pos = (newline - c->querybuf) + 2;
if (ll <= 0)
{
sdsrange (c->querybuf, pos, -1);
return 0;
}
}
while (c->multibulklen)
{
if (c->bulklen == -1)
{
newline = strchr (c->querybuf + pos, '\r');
if (newline == NULL)
{
if (sdslen (c->querybuf) > REDIS_INLINE_MAX_SIZE)
{
addReplyStr (c,
"-ERR Protocol error: too big bulk count string\r\n");
c->flags |= REDIS_CLOSE_AFTER_REPLY;
}
break;
}
if (newline - (c->querybuf) > (sdslen (c->querybuf) - 2))
break;
if (c->querybuf[pos] != '$')
{
addReplyStr (c, "-ERR Protocol error: expected '$', got '");
addReplyStrLen (c, c->querybuf + pos, 1);
addReplyStr (c, "'\r\n");
c->flags |= REDIS_CLOSE_AFTER_REPLY;
return -1;
}
ok =
string2ll (c->querybuf + pos + 1,
newline - (c->querybuf + pos + 1), &ll);
if (!ok || ll < 0 || ll > 512 * 1024 * 1024)
{
addReplyStr (c, "-ERR Protocol error: invalid bulk length\r\n");
c->flags |= REDIS_CLOSE_AFTER_REPLY;
return -1;
}
pos += newline - (c->querybuf + pos) + 2;
c->bulklen = ll;
}
if (sdslen (c->querybuf) - pos < (unsigned) (c->bulklen + 2))
break;
c->argvlen[c->argc] = c->bulklen;
c->argv[c->argc++] = sdsnewlen (c->querybuf + pos, c->bulklen);
pos += c->bulklen + 2;
c->bulklen = -1;
c->multibulklen--;
}
if (pos)
sdsrange (c->querybuf, pos, -1);
if (c->multibulklen)
return -1;
if (c->argc == 1 && sdslen (c->argv[0]) == 4
&& !strcasecmp (c->argv[0], "quit"))
{
addReplyStrLen (c, "+OK\r\n", 5);
c->flags |= REDIS_CLOSE_AFTER_REPLY;
return -1;
}
arc_append_commandcv (c->rqst, c->argc, (const char **) c->argv,
c->argvlen);
c->total_append_command++;
return 0;
}
static void *
clientThread (void *arg)
{
client_t *c = (client_t *) arg;
fd_set rfds, wfds;
int ret;
c->querybuf = sdsMakeRoomFor (sdsempty (), DEFAULT_QUERY_BUF_SIZE);
c->replybuf = sdsMakeRoomFor (sdsempty (), DEFAULT_QUERY_BUF_SIZE);
c->argc = 0;
c->argv = NULL;
c->argvlen = NULL;
c->reqtype = 0;
c->multibulklen = 0;
c->bulklen = -1;
c->rqst = arc_create_request ();
c->flags = 0;
c->total_append_command = 0;
FD_ZERO (&rfds);
FD_ZERO (&wfds);
while (1)
{
struct timeval timeout;
FD_CLR (c->fd, &rfds);
FD_CLR (c->fd, &wfds);
if (!(c->flags & REDIS_CLOSE_AFTER_REPLY))
FD_SET (c->fd, &rfds);
if (sdslen (c->replybuf) > 0)
FD_SET (c->fd, &wfds);
timeout.tv_sec = 1;
timeout.tv_usec = 0;
ret = select (c->fd + 1, &rfds, &wfds, NULL, &timeout);
if (ret == -1)
{
perror ("select");
freeClient (c);
}
if (server.shutdown_signal)
{
c->flags |= REDIS_CLOSE_AFTER_REPLY;
}
/* readable */
if (FD_ISSET (c->fd, &rfds))
{
int pos = sdslen (c->querybuf);
int avail = sdsavail (c->querybuf);
ssize_t nread;
if (avail == 0)
{
c->querybuf =
sdsMakeRoomFor (c->querybuf, sdslen (c->querybuf));
avail = sdsavail (c->querybuf);
}
nread = read (c->fd, c->querybuf + pos, avail);
if (nread > 0)
{
sdsIncrLen (c->querybuf, nread);
processInputBuffer (c);
if (c->total_append_command)
{
int arc_errno, arc_be_errno, be_errno;
arc_ref_t *arc_ref;
arc_ref = acquire_arc_ref ();
ret =
arc_do_request (arc_ref->arc, c->rqst,
server.query_timeout_millis, &be_errno);
if (ret == -1)
{
arc_errno = errno;
arc_be_errno = be_errno;
}
else
{
ret = processReply (c, &be_errno);
if (ret == -1)
{
arc_errno = errno;
arc_be_errno = be_errno;
}
}
arc_free_request (c->rqst);
release_arc_ref (arc_ref);
c->rqst = arc_create_request ();
if (ret == -1)
{
if (arc_errno == ARC_ERR_TIMEOUT
|| (arc_errno == ARC_ERR_BACKEND
&& arc_be_errno == ARC_ERR_TIMEOUT))
{
addReplyStr (c, "-ERR Redis Timeout\r\n");
}
else
{
addReplyStr (c, "-ERR Internal Error\r\n");
}
c->flags |= REDIS_CLOSE_AFTER_REPLY;
}
}
}
else
{
if (nread == -1 && errno == EAGAIN)
{
/* Skip */
}
else
{
freeClient (c);
}
}
}
/* writable */
if (FD_ISSET (c->fd, &wfds))
{
int pos = 0;
int avail = sdslen (c->replybuf);
ssize_t nwritten;
nwritten = write (c->fd, c->replybuf + pos, avail);
if (nwritten > 0)
{
avail -= nwritten;
pos += nwritten;
sdsrange (c->replybuf, pos, -1);
}
else
{
if (nwritten == -1 && errno == EAGAIN)
{
/* Skip */
}
else
{
freeClient (c);
}
}
}
if (sdslen (c->replybuf) == 0 && (c->flags & REDIS_CLOSE_AFTER_REPLY))
{
freeClient (c);
}
}
return NULL;
}
static int cliSendCommand(int argc, char **argv, int repeat) {
char *command = argv[0];
size_t *argvlen;
int j, output_raw;
if (context == NULL) return REDIS_ERR;
output_raw = 0;
if (!strcasecmp(command,"info") ||
(argc == 2 && !strcasecmp(command,"client") &&
!strcasecmp(argv[1],"list")))
{
output_raw = 1;
}
if (!strcasecmp(command,"help") || !strcasecmp(command,"?")) {
cliOutputHelp(--argc, ++argv);
return REDIS_OK;
}
if (!strcasecmp(command,"shutdown")) config.shutdown = 1;
if (!strcasecmp(command,"monitor")) config.monitor_mode = 1;
if (!strcasecmp(command,"subscribe") ||
!strcasecmp(command,"psubscribe")) config.pubsub_mode = 1;
/* Setup argument length */
argvlen = malloc(argc*sizeof(size_t));
for (j = 0; j < argc; j++)
argvlen[j] = sdslen(argv[j]);
while(repeat--) {
redisAppendCommandArgv(context,argc,(const char**)argv,argvlen);
while (config.monitor_mode) {
if (cliReadReply(output_raw) != REDIS_OK) exit(1);
fflush(stdout);
}
if (config.pubsub_mode) {
if (!config.raw_output)
printf("Reading messages... (press Ctrl-C to quit)\n");
while (1) {
if (cliReadReply(output_raw) != REDIS_OK) exit(1);
}
}
if (cliReadReply(output_raw) != REDIS_OK) {
free(argvlen);
return REDIS_ERR;
} else {
/* Store database number when SELECT was successfully executed. */
if (!strcasecmp(command,"select") && argc == 2) {
config.dbnum = atoi(argv[1]);
cliRefreshPrompt();
}
}
if (config.interval) usleep(config.interval);
fflush(stdout); /* Make it grep friendly */
}
free(argvlen);
return REDIS_OK;
}
static sds cliFormatReplyTTY(redisReply *r, char *prefix) {
sds out = sdsempty();
switch (r->type) {
case REDIS_REPLY_ERROR:
out = sdscatprintf(out,"(error) %s\n", r->str);
break;
case REDIS_REPLY_STATUS:
out = sdscat(out,r->str);
out = sdscat(out,"\n");
break;
case REDIS_REPLY_INTEGER:
out = sdscatprintf(out,"(integer) %lld\n",r->integer);
break;
case REDIS_REPLY_STRING:
/* If you are producing output for the standard output we want
* a more interesting output with quoted characters and so forth */
out = sdscatrepr(out,r->str,r->len);
out = sdscat(out,"\n");
break;
case REDIS_REPLY_NIL:
out = sdscat(out,"(nil)\n");
break;
case REDIS_REPLY_ARRAY:
if (r->elements == 0) {
out = sdscat(out,"(empty list or set)\n");
} else {
unsigned int i, idxlen = 0;
char _prefixlen[16];
char _prefixfmt[16];
sds _prefix;
sds tmp;
/* Calculate chars needed to represent the largest index */
i = r->elements;
do {
idxlen++;
i /= 10;
} while(i);
/* Prefix for nested multi bulks should grow with idxlen+2 spaces */
memset(_prefixlen,' ',idxlen+2);
_prefixlen[idxlen+2] = '\0';
_prefix = sdscat(sdsnew(prefix),_prefixlen);
/* Setup prefix format for every entry */
snprintf(_prefixfmt,sizeof(_prefixfmt),"%%s%%%dd) ",idxlen);
for (i = 0; i < r->elements; i++) {
/* Don't use the prefix for the first element, as the parent
* caller already prepended the index number. */
out = sdscatprintf(out,_prefixfmt,i == 0 ? "" : prefix,i+1);
/* Format the multi bulk entry */
tmp = cliFormatReplyTTY(r->element[i],_prefix);
out = sdscatlen(out,tmp,sdslen(tmp));
sdsfree(tmp);
}
sdsfree(_prefix);
}
break;
default:
fprintf(stderr,"Unknown reply type: %d\n", r->type);
exit(1);
}
return out;
}
/* Try to encode a string object in order to save space */
robj *tryObjectEncoding(robj *o) {
long value;
sds s = o->ptr;
size_t len;
/* Make sure this is a string object, the only type we encode
* in this function. Other types use encoded memory efficient
* representations but are handled by the commands implementing
* the type. */
assert(o->type == OBJ_STRING);
/* We try some specialized encoding only for objects that are
* RAW or EMBSTR encoded, in other words objects that are still
* in represented by an actually array of chars. */
if (!sdsEncodedObject(o)) return o;
/* It's not safe to encode shared objects: shared objects can be shared
* everywhere in the "object space" of Redis and may end in places where
* they are not handled. We handle them only as values in the keyspace. */
if (o->refcount > 1) return o;
/* Check if we can represent this string as a long integer.
* Note that we are sure that a string larger than 21 chars is not
* representable as a 32 nor 64 bit integer. */
len = sdslen(s);
if (len <= 21 && string2l(s,len,&value)) {
/* This object is encodable as a long. Try to use a shared object.
* Note that we avoid using shared integers when maxmemory is used
* because every object needs to have a private LRU field for the LRU
* algorithm to work well. */
if (server.max_memory == 0 &&
value >= 0 && value < OBJ_SHARED_INTEGERS)
{
decrRefCount(o);
incrRefCount(shared.integers[value]);
return shared.integers[value];
} else {
if (o->encoding == OBJ_ENCODING_RAW) sdsfree(o->ptr);
o->encoding = OBJ_ENCODING_INT;
o->ptr = (void*) value;
return o;
}
}
/* If the string is small and is still RAW encoded,
* try the EMBSTR encoding which is more efficient.
* In this representation the object and the SDS string are allocated
* in the same chunk of memory to save space and cache misses. */
if (len <= OBJ_ENCODING_EMBSTR_SIZE_LIMIT) {
robj *emb;
if (o->encoding == OBJ_ENCODING_EMBSTR) return o;
emb = createEmbeddedStringObject(s,sdslen(s));
decrRefCount(o);
return emb;
}
/* We can't encode the object...
*
* Do the last try, and at least optimize the SDS string inside
* the string object to require little space, in case there
* is more than 10% of free space at the end of the SDS string.
*
* We do that only for relatively large strings as this branch
* is only entered if the length of the string is greater than
* OBJ_ENCODING_EMBSTR_SIZE_LIMIT. */
if (o->encoding == OBJ_ENCODING_RAW &&
sdsavail(s) > len/10)
{
o->ptr = sdsRemoveFreeSpace(o->ptr);
}
/* Return the original object. */
return o;
}
/* Functions managing dictionary of callbacks for pub/sub. */
static unsigned int callbackHash(const void *key) {
return dictGenHashFunction((unsigned char*)key,sdslen((char*)key));
}
robj *dupStringObject(robj *o) {
redisAssertWithInfo(NULL,o,o->encoding == REDIS_ENCODING_RAW);
return createStringObject(o->ptr,sdslen(o->ptr));
}
void spopWithCountCommand(client *c) {
long l;
unsigned long count, size;
robj *set;
/* Get the count argument */
if (getLongFromObjectOrReply(c,c->argv[2],&l,NULL) != C_OK) return;
if (l >= 0) {
count = (unsigned) l;
} else {
addReply(c,shared.outofrangeerr);
return;
}
/* Make sure a key with the name inputted exists, and that it's type is
* indeed a set. Otherwise, return nil */
if ((set = lookupKeyReadOrReply(c,c->argv[1],shared.emptymultibulk))
== NULL || checkType(c,set,OBJ_SET)) return;
/* If count is zero, serve an empty multibulk ASAP to avoid special
* cases later. */
if (count == 0) {
addReply(c,shared.emptymultibulk);
return;
}
size = setTypeSize(set);
/* Generate an SPOP keyspace notification */
notifyKeyspaceEvent(NOTIFY_SET,"spop",c->argv[1],c->db->id);
server.dirty += count;
/* CASE 1:
* The number of requested elements is greater than or equal to
* the number of elements inside the set: simply return the whole set. */
if (count >= size) {
/* We just return the entire set */
sunionDiffGenericCommand(c,c->argv+1,1,NULL,SET_OP_UNION);
/* Delete the set as it is now empty */
dbDelete(c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],c->db->id);
/* Propagate this command as an DEL operation */
rewriteClientCommandVector(c,2,shared.del,c->argv[1]);
signalModifiedKey(c->db,c->argv[1]);
server.dirty++;
return;
}
/* Case 2 and 3 require to replicate SPOP as a set of SREM commands.
* Prepare our replication argument vector. Also send the array length
* which is common to both the code paths. */
robj *propargv[3];
propargv[0] = createStringObject("SREM",4);
propargv[1] = c->argv[1];
addReplyMultiBulkLen(c,count);
/* Common iteration vars. */
sds sdsele;
robj *objele;
int encoding;
int64_t llele;
unsigned long remaining = size-count; /* Elements left after SPOP. */
/* If we are here, the number of requested elements is less than the
* number of elements inside the set. Also we are sure that count < size.
* Use two different strategies.
*
* CASE 2: The number of elements to return is small compared to the
* set size. We can just extract random elements and return them to
* the set. */
if (remaining*SPOP_MOVE_STRATEGY_MUL > count) {
while(count--) {
/* Emit and remove. */
encoding = setTypeRandomElement(set,&sdsele,&llele);
if (encoding == OBJ_ENCODING_INTSET) {
addReplyBulkLongLong(c,llele);
objele = createStringObjectFromLongLong(llele);
set->ptr = intsetRemove(set->ptr,llele,NULL);
} else {
addReplyBulkCBuffer(c,sdsele,sdslen(sdsele));
objele = createStringObject(sdsele,sdslen(sdsele));
setTypeRemove(set,sdsele);
}
/* Replicate/AOF this command as an SREM operation */
propargv[2] = objele;
alsoPropagate(server.sremCommand,c->db->id,propargv,3,
PROPAGATE_AOF|PROPAGATE_REPL);
decrRefCount(objele);
}
} else {
/* CASE 3: The number of elements to return is very big, approaching
* the size of the set itself. After some time extracting random elements
* from such a set becomes computationally expensive, so we use
* a different strategy, we extract random elements that we don't
* want to return (the elements that will remain part of the set),
* creating a new set as we do this (that will be stored as the original
* set). Then we return the elements left in the original set and
* release it. */
robj *newset = NULL;
/* Create a new set with just the remaining elements. */
while(remaining--) {
encoding = setTypeRandomElement(set,&sdsele,&llele);
if (encoding == OBJ_ENCODING_INTSET) {
sdsele = sdsfromlonglong(llele);
} else {
sdsele = sdsdup(sdsele);
}
if (!newset) newset = setTypeCreate(sdsele);
setTypeAdd(newset,sdsele);
setTypeRemove(set,sdsele);
sdsfree(sdsele);
}
/* Assign the new set as the key value. */
incrRefCount(set); /* Protect the old set value. */
dbOverwrite(c->db,c->argv[1],newset);
/* Tranfer the old set to the client and release it. */
setTypeIterator *si;
si = setTypeInitIterator(set);
while((encoding = setTypeNext(si,&sdsele,&llele)) != -1) {
if (encoding == OBJ_ENCODING_INTSET) {
addReplyBulkLongLong(c,llele);
objele = createStringObjectFromLongLong(llele);
} else {
addReplyBulkCBuffer(c,sdsele,sdslen(sdsele));
objele = createStringObject(sdsele,sdslen(sdsele));
}
/* Replicate/AOF this command as an SREM operation */
propargv[2] = objele;
alsoPropagate(server.sremCommand,c->db->id,propargv,3,
PROPAGATE_AOF|PROPAGATE_REPL);
decrRefCount(objele);
}
setTypeReleaseIterator(si);
decrRefCount(set);
}
/* Don't propagate the command itself even if we incremented the
* dirty counter. We don't want to propagate an SPOP command since
* we propagated the command as a set of SREMs operations using
* the alsoPropagate() API. */
decrRefCount(propargv[0]);
preventCommandPropagation(c);
signalModifiedKey(c->db,c->argv[1]);
server.dirty++;
}
/* This function is similar to sdscatprintf, but much faster as it does
* not rely on sprintf() family functions implemented by the libc that
* are often very slow. Moreover directly handling the sds string as
* new data is concatenated provides a performance improvement.
*
* However this function only handles an incompatible subset of printf-alike
* format specifiers:
*
* %s - C String
* %S - SDS string
* %i - signed int
* %I - 64 bit signed integer (long long, int64_t)
* %u - unsigned int
* %U - 64 bit unsigned integer (unsigned long long, uint64_t)
* %% - Verbatim "%" character.
*/
sds sdscatfmt(sds s, char const *fmt, ...) {
struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr)));
size_t initlen = sdslen(s);
const char *f = fmt;
int i;
va_list ap;
va_start(ap,fmt);
f = fmt; /* Next format specifier byte to process. */
i = initlen; /* Position of the next byte to write to dest str. */
while(*f) {
char next, *str;
unsigned int l;
long long num;
unsigned long long unum;
/* Make sure there is always space for at least 1 char. */
if (sh->free == 0) {
s = sdsMakeRoomFor(s,1);
sh = (void*) (s-(sizeof(struct sdshdr)));
}
switch(*f) {
case '%':
next = *(f+1);
f++;
switch(next) {
case 's':
case 'S':
str = va_arg(ap,char*);
l = (next == 's') ? strlen(str) : sdslen(str);
if (sh->free < l) {
s = sdsMakeRoomFor(s,l);
sh = (void*) (s-(sizeof(struct sdshdr)));
}
memcpy(s+i,str,l);
sh->len += l;
sh->free -= l;
i += l;
break;
case 'i':
case 'I':
if (next == 'i')
num = va_arg(ap,int);
else
num = va_arg(ap,long long);
{
char buf[SDS_LLSTR_SIZE];
l = sdsll2str(buf,num);
if (sh->free < l) {
s = sdsMakeRoomFor(s,l);
sh = (void*) (s-(sizeof(struct sdshdr)));
}
memcpy(s+i,buf,l);
sh->len += l;
sh->free -= l;
i += l;
}
break;
case 'u':
case 'U':
if (next == 'u')
unum = va_arg(ap,unsigned int);
else
unum = va_arg(ap,unsigned long long);
{
char buf[SDS_LLSTR_SIZE];
l = sdsull2str(buf,unum);
if (sh->free < l) {
s = sdsMakeRoomFor(s,l);
sh = (void*) (s-(sizeof(struct sdshdr)));
}
memcpy(s+i,buf,l);
sh->len += l;
sh->free -= l;
i += l;
}
break;
default: /* Handle %% and generally %<unknown>. */
s[i++] = next;
sh->len += 1;
sh->free -= 1;
break;
}
break;
default:
s[i++] = *f;
sh->len += 1;
sh->free -= 1;
break;
}
void srandmemberWithCountCommand(client *c) {
long l;
unsigned long count, size;
int uniq = 1;
robj *set;
sds ele;
int64_t llele;
int encoding;
dict *d;
if (getLongFromObjectOrReply(c,c->argv[2],&l,NULL) != C_OK) return;
if (l >= 0) {
count = (unsigned) l;
} else {
/* A negative count means: return the same elements multiple times
* (i.e. don't remove the extracted element after every extraction). */
count = -l;
uniq = 0;
}
if ((set = lookupKeyReadOrReply(c,c->argv[1],shared.emptymultibulk))
== NULL || checkType(c,set,OBJ_SET)) return;
size = setTypeSize(set);
/* If count is zero, serve it ASAP to avoid special cases later. */
if (count == 0) {
addReply(c,shared.emptymultibulk);
return;
}
/* CASE 1: The count was negative, so the extraction method is just:
* "return N random elements" sampling the whole set every time.
* This case is trivial and can be served without auxiliary data
* structures. */
if (!uniq) {
addReplyMultiBulkLen(c,count);
while(count--) {
encoding = setTypeRandomElement(set,&ele,&llele);
if (encoding == OBJ_ENCODING_INTSET) {
addReplyBulkLongLong(c,llele);
} else {
addReplyBulkCBuffer(c,ele,sdslen(ele));
}
}
return;
}
/* CASE 2:
* The number of requested elements is greater than the number of
* elements inside the set: simply return the whole set. */
if (count >= size) {
sunionDiffGenericCommand(c,c->argv+1,1,NULL,SET_OP_UNION);
return;
}
/* For CASE 3 and CASE 4 we need an auxiliary dictionary. */
d = dictCreate(&objectKeyPointerValueDictType,NULL);
/* CASE 3:
* The number of elements inside the set is not greater than
* SRANDMEMBER_SUB_STRATEGY_MUL times the number of requested elements.
* In this case we create a set from scratch with all the elements, and
* subtract random elements to reach the requested number of elements.
*
* This is done because if the number of requsted elements is just
* a bit less than the number of elements in the set, the natural approach
* used into CASE 3 is highly inefficient. */
if (count*SRANDMEMBER_SUB_STRATEGY_MUL > size) {
setTypeIterator *si;
/* Add all the elements into the temporary dictionary. */
si = setTypeInitIterator(set);
while((encoding = setTypeNext(si,&ele,&llele)) != -1) {
int retval = DICT_ERR;
if (encoding == OBJ_ENCODING_INTSET) {
retval = dictAdd(d,createStringObjectFromLongLong(llele),NULL);
} else {
retval = dictAdd(d,createStringObject(ele,sdslen(ele)),NULL);
}
serverAssert(retval == DICT_OK);
}
setTypeReleaseIterator(si);
serverAssert(dictSize(d) == size);
/* Remove random elements to reach the right count. */
while(size > count) {
dictEntry *de;
de = dictGetRandomKey(d);
dictDelete(d,dictGetKey(de));
size--;
}
}
/* CASE 4: We have a big set compared to the requested number of elements.
* In this case we can simply get random elements from the set and add
* to the temporary set, trying to eventually get enough unique elements
* to reach the specified count. */
else {
unsigned long added = 0;
robj *objele;
while(added < count) {
encoding = setTypeRandomElement(set,&ele,&llele);
if (encoding == OBJ_ENCODING_INTSET) {
objele = createStringObjectFromLongLong(llele);
} else {
objele = createStringObject(ele,sdslen(ele));
}
/* Try to add the object to the dictionary. If it already exists
* free it, otherwise increment the number of objects we have
* in the result dictionary. */
if (dictAdd(d,objele,NULL) == DICT_OK)
added++;
else
decrRefCount(objele);
}
}
/* CASE 3 & 4: send the result to the user. */
{
dictIterator *di;
dictEntry *de;
addReplyMultiBulkLen(c,count);
di = dictGetIterator(d);
while((de = dictNext(di)) != NULL)
addReplyBulk(c,dictGetKey(de));
dictReleaseIterator(di);
dictRelease(d);
}
}
static int send_msg_to_all(check_unit *cunit, struct msg *msg)
{
int ret;
thread_data *cdata = cunit->cdata;
redis_group *trgroup = cdata->trgroup;
redis_node *trnode;
struct msg *msg_same;
listNode *lnode;
struct mbuf *mbuf;
if (cunit == NULL || msg == NULL) {
return RMT_ERROR;
}
msg_same = msg_get(msg->mb, msg->request, msg->kind);
if (msg_same == NULL) {
log_error("ERROR: msg clone failed.");
msg_put(msg);
msg_free(msg);
msg = NULL;
return RMT_ERROR;
}
lnode = listFirst(msg->data);
while (lnode) {
mbuf = listNodeValue(lnode);
lnode = lnode->next;
ret = msg_append_full(msg_same, mbuf->pos, mbuf_length(mbuf));
if (ret != RMT_OK) {
log_error("ERROR: out of memory.");
msg_put(msg_same);
msg_free(msg_same);
msg = NULL;
return RMT_ERROR;
}
}
msg_same->ptr = msg->ptr;
msg_same->resp_check = msg->resp_check;
ret = prepare_send_msg(cunit->srnode, msg, cunit->srnode);
if (ret != RMT_OK) {
msg_put(msg);
msg_free(msg);
msg = NULL;
msg_put(msg_same);
msg_free(msg_same);
return RMT_ERROR;
}
msg = NULL;
trnode = trgroup->get_backend_node(trgroup, (uint8_t *)cunit->key, (uint32_t)sdslen(cunit->key));
if(prepare_send_msg(trnode, msg_same, trnode) != RMT_OK){
msg_put(msg_same);
msg_free(msg_same);
return RMT_ERROR;
}
return RMT_OK;
}
void sinterGenericCommand(client *c, robj **setkeys,
unsigned long setnum, robj *dstkey) {
robj **sets = zmalloc(sizeof(robj*)*setnum);
setTypeIterator *si;
robj *dstset = NULL;
sds elesds;
int64_t intobj;
void *replylen = NULL;
unsigned long j, cardinality = 0;
int encoding;
for (j = 0; j < setnum; j++) {
robj *setobj = dstkey ?
lookupKeyWrite(c->db,setkeys[j]) :
lookupKeyRead(c->db,setkeys[j]);
if (!setobj) {
zfree(sets);
if (dstkey) {
if (dbDelete(c->db,dstkey)) {
signalModifiedKey(c->db,dstkey);
server.dirty++;
}
addReply(c,shared.czero);
} else {
addReply(c,shared.emptymultibulk);
}
return;
}
if (checkType(c,setobj,OBJ_SET)) {
zfree(sets);
return;
}
sets[j] = setobj;
}
/* Sort sets from the smallest to largest, this will improve our
* algorithm's performance */
qsort(sets,setnum,sizeof(robj*),qsortCompareSetsByCardinality);
/* The first thing we should output is the total number of elements...
* since this is a multi-bulk write, but at this stage we don't know
* the intersection set size, so we use a trick, append an empty object
* to the output list and save the pointer to later modify it with the
* right length */
if (!dstkey) {
replylen = addDeferredMultiBulkLength(c);
} else {
/* If we have a target key where to store the resulting set
* create this key with an empty set inside */
dstset = createIntsetObject();
}
/* Iterate all the elements of the first (smallest) set, and test
* the element against all the other sets, if at least one set does
* not include the element it is discarded */
si = setTypeInitIterator(sets[0]);
while((encoding = setTypeNext(si,&elesds,&intobj)) != -1) {
for (j = 1; j < setnum; j++) {
if (sets[j] == sets[0]) continue;
if (encoding == OBJ_ENCODING_INTSET) {
/* intset with intset is simple... and fast */
if (sets[j]->encoding == OBJ_ENCODING_INTSET &&
!intsetFind((intset*)sets[j]->ptr,intobj))
{
break;
/* in order to compare an integer with an object we
* have to use the generic function, creating an object
* for this */
} else if (sets[j]->encoding == OBJ_ENCODING_HT) {
elesds = sdsfromlonglong(intobj);
if (!setTypeIsMember(sets[j],elesds)) {
sdsfree(elesds);
break;
}
sdsfree(elesds);
}
} else if (encoding == OBJ_ENCODING_HT) {
if (!setTypeIsMember(sets[j],elesds)) {
break;
}
}
}
/* Only take action when all sets contain the member */
if (j == setnum) {
if (!dstkey) {
if (encoding == OBJ_ENCODING_HT)
addReplyBulkCBuffer(c,elesds,sdslen(elesds));
else
addReplyBulkLongLong(c,intobj);
cardinality++;
} else {
if (encoding == OBJ_ENCODING_INTSET) {
elesds = sdsfromlonglong(intobj);
setTypeAdd(dstset,elesds);
sdsfree(elesds);
} else {
setTypeAdd(dstset,elesds);
}
}
}
}
setTypeReleaseIterator(si);
if (dstkey) {
/* Store the resulting set into the target, if the intersection
* is not an empty set. */
int deleted = dbDelete(c->db,dstkey);
if (setTypeSize(dstset) > 0) {
dbAdd(c->db,dstkey,dstset);
addReplyLongLong(c,setTypeSize(dstset));
notifyKeyspaceEvent(NOTIFY_SET,"sinterstore",
dstkey,c->db->id);
} else {
decrRefCount(dstset);
addReply(c,shared.czero);
if (deleted)
notifyKeyspaceEvent(NOTIFY_GENERIC,"del",
dstkey,c->db->id);
}
signalModifiedKey(c->db,dstkey);
server.dirty++;
} else {
setDeferredMultiBulkLength(c,replylen,cardinality);
}
zfree(sets);
}
int redisvFormatCommand(char **target, const char *format, va_list ap) {
const char *c = format;
char *cmd = NULL; /* final command */
int pos; /* position in final command */
sds curarg, newarg; /* current argument */
int touched = 0; /* was the current argument touched? */
char **curargv = NULL, **newargv = NULL;
int argc = 0;
int totlen = 0;
int j;
/* Abort if there is not target to set */
if (target == NULL)
return -1;
/* Build the command string accordingly to protocol */
curarg = sdsempty();
if (curarg == NULL)
return -1;
while(*c != '\0') {
if (*c != '%' || c[1] == '\0') {
if (*c == ' ') {
if (touched) {
newargv = realloc(curargv,sizeof(char*)*(argc+1));
if (newargv == NULL) goto err;
curargv = newargv;
curargv[argc++] = curarg;
totlen += bulklen(sdslen(curarg));
/* curarg is put in argv so it can be overwritten. */
curarg = sdsempty();
if (curarg == NULL) goto err;
touched = 0;
}
} else {
newarg = sdscatlen(curarg,c,1);
if (newarg == NULL) goto err;
curarg = newarg;
touched = 1;
}
} else {
char *arg;
size_t size;
/* Set newarg so it can be checked even if it is not touched. */
newarg = curarg;
switch(c[1]) {
case 's':
arg = va_arg(ap,char*);
size = strlen(arg);
if (size > 0)
newarg = sdscatlen(curarg,arg,size);
break;
case 'b':
arg = va_arg(ap,char*);
size = va_arg(ap,size_t);
if (size > 0)
newarg = sdscatlen(curarg,arg,size);
break;
case '%':
newarg = sdscat(curarg,"%");
break;
default:
/* Try to detect printf format */
{
static const char intfmts[] = "diouxX";
char _format[16];
const char *_p = c+1;
size_t _l = 0;
va_list _cpy;
/* Flags */
if (*_p != '\0' && *_p == '#') _p++;
if (*_p != '\0' && *_p == '0') _p++;
if (*_p != '\0' && *_p == '-') _p++;
if (*_p != '\0' && *_p == ' ') _p++;
if (*_p != '\0' && *_p == '+') _p++;
/* Field width */
while (*_p != '\0' && isdigit(*_p)) _p++;
/* Precision */
if (*_p == '.') {
_p++;
while (*_p != '\0' && isdigit(*_p)) _p++;
}
/* Copy va_list before consuming with va_arg */
va_copy(_cpy,ap);
/* Integer conversion (without modifiers) */
if (strchr(intfmts,*_p) != NULL) {
va_arg(ap,int);
goto fmt_valid;
}
/* Double conversion (without modifiers) */
if (strchr("eEfFgGaA",*_p) != NULL) {
va_arg(ap,double);
goto fmt_valid;
}
/* Size: char */
if (_p[0] == 'h' && _p[1] == 'h') {
_p += 2;
if (*_p != '\0' && strchr(intfmts,*_p) != NULL) {
va_arg(ap,int); /* char gets promoted to int */
goto fmt_valid;
}
goto fmt_invalid;
}
/* Size: short */
if (_p[0] == 'h') {
_p += 1;
if (*_p != '\0' && strchr(intfmts,*_p) != NULL) {
va_arg(ap,int); /* short gets promoted to int */
goto fmt_valid;
}
goto fmt_invalid;
}
/* Size: long long */
if (_p[0] == 'l' && _p[1] == 'l') {
_p += 2;
if (*_p != '\0' && strchr(intfmts,*_p) != NULL) {
va_arg(ap,long long);
goto fmt_valid;
}
goto fmt_invalid;
}
/* Size: long */
if (_p[0] == 'l') {
_p += 1;
if (*_p != '\0' && strchr(intfmts,*_p) != NULL) {
va_arg(ap,long);
goto fmt_valid;
}
goto fmt_invalid;
}
fmt_invalid:
va_end(_cpy);
goto err;
fmt_valid:
_l = (_p+1)-c;
if (_l < sizeof(_format)-2) {
memcpy(_format,c,_l);
_format[_l] = '\0';
newarg = sdscatvprintf(curarg,_format,_cpy);
/* Update current position (note: outer blocks
* increment c twice so compensate here) */
c = _p-1;
}
va_end(_cpy);
break;
}
}
if (newarg == NULL) goto err;
curarg = newarg;
touched = 1;
c++;
}
c++;
}
/* Add the last argument if needed */
if (touched) {
newargv = realloc(curargv,sizeof(char*)*(argc+1));
if (newargv == NULL) goto err;
curargv = newargv;
curargv[argc++] = curarg;
totlen += bulklen(sdslen(curarg));
} else {
sdsfree(curarg);
}
/* Clear curarg because it was put in curargv or was free'd. */
curarg = NULL;
/* Add bytes needed to hold multi bulk count */
totlen += 1+intlen(argc)+2;
/* Build the command at protocol level */
cmd = malloc(totlen+1);
if (cmd == NULL) goto err;
pos = sprintf(cmd,"*%d\r\n",argc);
for (j = 0; j < argc; j++) {
pos += sprintf(cmd+pos,"$%zu\r\n",sdslen(curargv[j]));
memcpy(cmd+pos,curargv[j],sdslen(curargv[j]));
pos += sdslen(curargv[j]);
sdsfree(curargv[j]);
cmd[pos++] = '\r';
cmd[pos++] = '\n';
}
assert(pos == totlen);
cmd[pos] = '\0';
free(curargv);
*target = cmd;
return totlen;
err:
while(argc--)
sdsfree(curargv[argc]);
free(curargv);
if (curarg != NULL)
sdsfree(curarg);
/* No need to check cmd since it is the last statement that can fail,
* but do it anyway to be as defensive as possible. */
if (cmd != NULL)
free(cmd);
return -1;
}
void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum,
robj *dstkey, int op) {
robj **sets = zmalloc(sizeof(robj*)*setnum);
setTypeIterator *si;
robj *dstset = NULL;
sds ele;
int j, cardinality = 0;
int diff_algo = 1;
for (j = 0; j < setnum; j++) {
robj *setobj = dstkey ?
lookupKeyWrite(c->db,setkeys[j]) :
lookupKeyRead(c->db,setkeys[j]);
if (!setobj) {
sets[j] = NULL;
continue;
}
if (checkType(c,setobj,OBJ_SET)) {
zfree(sets);
return;
}
sets[j] = setobj;
}
/* Select what DIFF algorithm to use.
*
* Algorithm 1 is O(N*M) where N is the size of the element first set
* and M the total number of sets.
*
* Algorithm 2 is O(N) where N is the total number of elements in all
* the sets.
*
* We compute what is the best bet with the current input here. */
if (op == SET_OP_DIFF && sets[0]) {
long long algo_one_work = 0, algo_two_work = 0;
for (j = 0; j < setnum; j++) {
if (sets[j] == NULL) continue;
algo_one_work += setTypeSize(sets[0]);
algo_two_work += setTypeSize(sets[j]);
}
/* Algorithm 1 has better constant times and performs less operations
* if there are elements in common. Give it some advantage. */
algo_one_work /= 2;
diff_algo = (algo_one_work <= algo_two_work) ? 1 : 2;
if (diff_algo == 1 && setnum > 1) {
/* With algorithm 1 it is better to order the sets to subtract
* by decreasing size, so that we are more likely to find
* duplicated elements ASAP. */
qsort(sets+1,setnum-1,sizeof(robj*),
qsortCompareSetsByRevCardinality);
}
}
/* We need a temp set object to store our union. If the dstkey
* is not NULL (that is, we are inside an SUNIONSTORE operation) then
* this set object will be the resulting object to set into the target key*/
dstset = createIntsetObject();
if (op == SET_OP_UNION) {
/* Union is trivial, just add every element of every set to the
* temporary set. */
for (j = 0; j < setnum; j++) {
if (!sets[j]) continue; /* non existing keys are like empty sets */
si = setTypeInitIterator(sets[j]);
while((ele = setTypeNextObject(si)) != NULL) {
if (setTypeAdd(dstset,ele)) cardinality++;
sdsfree(ele);
}
setTypeReleaseIterator(si);
}
} else if (op == SET_OP_DIFF && sets[0] && diff_algo == 1) {
/* DIFF Algorithm 1:
*
* We perform the diff by iterating all the elements of the first set,
* and only adding it to the target set if the element does not exist
* into all the other sets.
*
* This way we perform at max N*M operations, where N is the size of
* the first set, and M the number of sets. */
si = setTypeInitIterator(sets[0]);
while((ele = setTypeNextObject(si)) != NULL) {
for (j = 1; j < setnum; j++) {
if (!sets[j]) continue; /* no key is an empty set. */
if (sets[j] == sets[0]) break; /* same set! */
if (setTypeIsMember(sets[j],ele)) break;
}
if (j == setnum) {
/* There is no other set with this element. Add it. */
setTypeAdd(dstset,ele);
cardinality++;
}
sdsfree(ele);
}
setTypeReleaseIterator(si);
} else if (op == SET_OP_DIFF && sets[0] && diff_algo == 2) {
/* DIFF Algorithm 2:
*
* Add all the elements of the first set to the auxiliary set.
* Then remove all the elements of all the next sets from it.
*
* This is O(N) where N is the sum of all the elements in every
* set. */
for (j = 0; j < setnum; j++) {
if (!sets[j]) continue; /* non existing keys are like empty sets */
si = setTypeInitIterator(sets[j]);
while((ele = setTypeNextObject(si)) != NULL) {
if (j == 0) {
if (setTypeAdd(dstset,ele)) cardinality++;
} else {
if (setTypeRemove(dstset,ele)) cardinality--;
}
sdsfree(ele);
}
setTypeReleaseIterator(si);
/* Exit if result set is empty as any additional removal
* of elements will have no effect. */
if (cardinality == 0) break;
}
}
/* Output the content of the resulting set, if not in STORE mode */
if (!dstkey) {
addReplyMultiBulkLen(c,cardinality);
si = setTypeInitIterator(dstset);
while((ele = setTypeNextObject(si)) != NULL) {
addReplyBulkCBuffer(c,ele,sdslen(ele));
sdsfree(ele);
}
setTypeReleaseIterator(si);
decrRefCount(dstset);
} else {
/* If we have a target key where to store the resulting set
* create this key with the result set inside */
int deleted = dbDelete(c->db,dstkey);
if (setTypeSize(dstset) > 0) {
dbAdd(c->db,dstkey,dstset);
addReplyLongLong(c,setTypeSize(dstset));
notifyKeyspaceEvent(NOTIFY_SET,
op == SET_OP_UNION ? "sunionstore" : "sdiffstore",
dstkey,c->db->id);
} else {
decrRefCount(dstset);
addReply(c,shared.czero);
if (deleted)
notifyKeyspaceEvent(NOTIFY_GENERIC,"del",
dstkey,c->db->id);
}
signalModifiedKey(c->db,dstkey);
server.dirty++;
}
zfree(sets);
}
sds sdsdup(const sds s) {
return sdsnewlen(s, sdslen(s));
}
static void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int argc) {
sds buf = sdsempty();
int j;
ssize_t nwritten;
time_t now;
robj *tmpargv[3];
/* The DB this command was targetting is not the same as the last command
* we appendend. To issue a SELECT command is needed. */
if (dictid != server.appendseldb) {
char seldb[64];
snprintf(seldb,sizeof(seldb),"%d",dictid);
buf = sdscatprintf(buf,"*2\r\n$6\r\nSELECT\r\n$%lu\r\n%s\r\n",
(unsigned long)strlen(seldb), seldb);
server.appendseldb = dictid;
}
/* "Fix" the argv vector if the command is EXPIRE. We want to translate
* EXPIREs into EXPIREATs calls */
if (cmd->proc == expireCommand) {
long when;
tmpargv[0] = createStringObject("EXPIREAT",8);
tmpargv[1] = argv[1];
incrRefCount(argv[1]);
when = time(NULL)+strtol(argv[2]->ptr,NULL,10);
tmpargv[2] = createObject(REDIS_STRING,
sdscatprintf(sdsempty(),"%ld",when));
argv = tmpargv;
}
/* Append the actual command */
buf = sdscatprintf(buf,"*%d\r\n",argc);
for (j = 0; j < argc; j++) {
robj *o = argv[j];
o = getDecodedObject(o);
buf = sdscatprintf(buf,"$%lu\r\n",(unsigned long)sdslen(o->ptr));
buf = sdscatlen(buf,o->ptr,sdslen(o->ptr));
buf = sdscatlen(buf,"\r\n",2);
decrRefCount(o);
}
/* Free the objects from the modified argv for EXPIREAT */
if (cmd->proc == expireCommand) {
for (j = 0; j < 3; j++)
decrRefCount(argv[j]);
}
/* We want to perform a single write. This should be guaranteed atomic
* at least if the filesystem we are writing is a real physical one.
* While this will save us against the server being killed I don't think
* there is much to do about the whole server stopping for power problems
* or alike */
nwritten = write(server.appendfd,buf,sdslen(buf));
/* If a background append only file rewriting is in progress we want to
* accumulate the differences between the child DB and the current one
* in a buffer, so that when the child process will do its work we
* can append the differences to the new append only file. */
if (server.bgrewritechildpid != -1)
server.bgrewritebuf = sdscatlen(server.bgrewritebuf,buf,sdslen(buf));
sdsfree(buf);
now = time(NULL);
if (server.appendfsync == APPENDFSYNC_ALWAYS ||
(server.appendfsync == APPENDFSYNC_EVERYSEC &&
now-server.lastfsync > 1))
{
fsync(server.appendfd); /* Let's try to get this data on the disk */
server.lastfsync = now;
}
}
