int guts(int accept_fd, int listen_fd) { char buffer[RECV_WINDOW] = ""; // recv buffer int msgbuflen = MSG_SIZE; char status_msg[MSG_SIZE]; char *msg; // Incoming message. char *send_msg; // Outgoing message. char *tmp_msg; void *msg_cursor; struct response_struct response; int msglen = 0; // length of the assembled message that we receive. int recvlen = 0; // how many bytes recv call returns. int responselen = 0; int offset; int retval; char* token_vector[MAX_ARGS] = {'\0'}; int token_count = 0; // Re-register the sigterm handler to our cleanup function. signal(SIGTERM, sigterm_handler_child); close(listen_fd); // Close this resource from our parent. We don't need it any more. while (1) { msglen = 0; msgbuflen = MSG_SIZE; msg = malloc(sizeof(char) * msgbuflen); msg_cursor = (void*)msg; bzero(msg, msgbuflen); // Wait for some data while (((recvlen = recv(accept_fd, (void*)buffer, RECV_WINDOW, MSG_PEEK)) == -1) && (errno == EAGAIN)); if (recvlen == 0) { fprintf(stderr, "Client closed the connection.\n"); close(accept_fd); cleanup_and_exit(0); }; // Receive data from our buffered stream until we would block. while ((recvlen = recv(accept_fd, (void*)buffer, RECV_WINDOW, 0)) != -1) { if (recvlen == 0) { fprintf(stderr, "Client closed the connection.\n"); close(accept_fd); cleanup_and_exit(0); }; if (recvlen == -1) { fprintf(stderr, "Got error %d from recv.\n", errno); close(accept_fd); cleanup_and_exit(-1); }; // Extend our message buffer if need be. if ((msglen += recvlen) > (msgbuflen)) { msgbuflen += msgbuflen; offset = msg_cursor - (void*)msg; tmp_msg = malloc(sizeof(char) * msgbuflen); bzero(tmp_msg, msgbuflen); memcpy(tmp_msg, msg, offset); msg_cursor = tmp_msg + offset; free(msg); msg = tmp_msg; fprintf(stderr, "msgbuflen expanded to %d\n", msgbuflen); } memcpy(msg_cursor, (void*)buffer, recvlen); msg_cursor += recvlen; if (memchr((void*)buffer, '\n', recvlen)) break; // Got a terminator character. Go process our message. } tmp_msg = msg; strsep(&tmp_msg, "\r\n"); token_count = tokenize_command(msg, token_vector); switch (extract_command(token_vector, token_count)) { case 0: // quit cleanup_and_exit(); break; case 1: // create response = create_command(token_vector, token_count); break; case 2: // read response = read_command(token_vector, token_count); break; case 3: // delete response = delete_command(token_vector, token_count); break; case 4: // subkeys response = keys_command(token_vector, token_count); break; default: if ((response.msg = malloc(sizeof(char) * MSG_SIZE)) == NULL) { perror(NULL); cleanup_and_exit; } bzero(response.msg, MSG_SIZE); sprintf(response.msg, "Unknown command."); response.status = 1; } responselen = prepare_send_msg(response, &send_msg); if((send(accept_fd, (void*)send_msg, responselen, 0) == -1)) perror("Send failed"); free(msg); free(response.msg); free(send_msg); }; return(0); }
static int check_response(redis_node *rnode, struct msg *r) { int ret; rmtContext *ctx = rnode->ctx; struct msg *resp, *msg = NULL; check_data *chdata; check_unit *cunit; char extra_err[50]; struct array args; sds *arg; struct array *bulks1 = NULL, *bulks2 = NULL; sds *bulk1, *bulk2; if (r == NULL) { return RMT_ERROR; } extra_err[0] = '\0'; resp = r->peer; r->peer = NULL; array_init(&args, 3, sizeof(sds)); ASSERT(r->request && r->sent); ASSERT(resp != NULL && resp->request == 0); cunit = (check_unit *)r->ptr; chdata = cunit->cdata->data; if(resp->type == MSG_RSP_REDIS_ERROR){ log_warn("Response from node[%s] for %s is error", rnode->addr, msg_type_string(r->type)); goto error; } if (cunit->state == CHECK_UNIT_STATE_GET_KEY) { ASSERT(cunit->key == NULL); ASSERT(cunit->key_type == -1); ASSERT(cunit->result1 == NULL && cunit->result2 == NULL); ASSERT(cunit->srnode == rnode); if (resp->type != MSG_RSP_REDIS_BULK) { log_error("ERROR: the response type for command 'randomkey' from node[%s] is error: %s", rnode->addr, msg_type_string(resp->type)); goto error; } if (msg_cmp_str(resp, (const uint8_t*)REDIS_REPLY_BULK_NULL, rmt_strlen(REDIS_REPLY_BULK_NULL)) == 0) { /* source group may have no keys, stop it */ cunit->cdata->keys_count --; goto done; } cunit->key = redis_msg_response_get_bulk_string(resp); if (cunit->key == NULL) { log_error("ERROR: get bulk string from response of node[%s] failed, " "bulk_len: %"PRIu32", bulk_start: %p", rnode->addr, resp->bulk_len, resp->bulk_start); goto error; } if (ctx->filter != NULL && !stringmatchlen(ctx->filter, sdslen(ctx->filter), cunit->key, sdslen(cunit->key), 0)) { goto done; } ASSERT(sdslen(cunit->key) == resp->bulk_len); msg = msg_get(r->mb, 1, REDIS_DATA_TYPE_CMD); if (msg == NULL) { log_error("ERROR: out of memory."); goto error; } arg = array_push(&args); *arg = sdsnew("type"); arg = array_push(&args); *arg = sdsdup(cunit->key); ret = redis_msg_append_command_full_safe(msg, &args); if (ret != RMT_OK) { log_error("ERROR: msg append multi bulk len failed."); goto error; } while (array_n(&args) > 0) { arg = array_pop(&args); sdsfree(*arg); } msg->ptr = cunit; msg->resp_check = check_response; ret = prepare_send_msg(rnode, msg, rnode); if (ret != RMT_OK) { log_error("ERROR: prepare send msg node[%s] failed.", rnode->addr); goto error; } cunit->state = CHECK_UNIT_STATE_GET_TYPE; goto next_step; } if (cunit->state == CHECK_UNIT_STATE_GET_TYPE) { ASSERT(cunit->key != NULL); ASSERT(cunit->key_type == -1); ASSERT(cunit->result1 == NULL && cunit->result2 == NULL); ASSERT(cunit->srnode == rnode); if (resp->type != MSG_RSP_REDIS_STATUS) { log_error("ERROR: the response type for command 'type' from node[%s] is error: %s", rnode->addr, msg_type_string(resp->type)); goto error; } if (msg_cmp_str(resp, (const uint8_t*)REDIS_REPLY_STATUS_NONE, rmt_strlen(REDIS_REPLY_STATUS_NONE)) == 0) { /* This key doesn't exit, may be expired or evicted */ goto done; } msg = msg_get(r->mb, 1, REDIS_DATA_TYPE_CMD); if (msg == NULL) { log_error("ERROR: out of memory."); goto error; } if (msg_cmp_str(resp, (const uint8_t*)REDIS_REPLY_STATUS_STRING, rmt_strlen(REDIS_REPLY_STATUS_STRING)) == 0) { cunit->key_type = REDIS_STRING; arg = array_push(&args); *arg = sdsnew("get"); arg = array_push(&args); *arg = sdsdup(cunit->key); ret = redis_msg_append_command_full_safe(msg, &args); if (ret != RMT_OK) { log_error("ERROR: msg append multi bulk len failed."); goto error; } while (array_n(&args) > 0) { arg = array_pop(&args); sdsfree(*arg); } } else if (msg_cmp_str(resp, (const uint8_t*)REDIS_REPLY_STATUS_LIST, rmt_strlen(REDIS_REPLY_STATUS_LIST)) == 0) { cunit->key_type = REDIS_LIST; arg = array_push(&args); *arg = sdsnew("lrange"); arg = array_push(&args); *arg = sdsdup(cunit->key); arg = array_push(&args); *arg = sdsnew("0"); arg = array_push(&args); *arg = sdsnew("-1"); ret = redis_msg_append_command_full_safe(msg, &args); if (ret != RMT_OK) { log_error("ERROR: msg append multi bulk len failed."); goto error; } while (array_n(&args) > 0) { arg = array_pop(&args); sdsfree(*arg); } } else if (msg_cmp_str(resp, (const uint8_t*)REDIS_REPLY_STATUS_SET, rmt_strlen(REDIS_REPLY_STATUS_SET)) == 0) { cunit->key_type = REDIS_SET; arg = array_push(&args); *arg = sdsnew("smembers"); arg = array_push(&args); *arg = sdsdup(cunit->key); ret = redis_msg_append_command_full_safe(msg, &args); if (ret != RMT_OK) { log_error("ERROR: msg append multi bulk len failed."); goto error; } while (array_n(&args) > 0) { arg = array_pop(&args); sdsfree(*arg); } } else if (msg_cmp_str(resp, (const uint8_t*)REDIS_REPLY_STATUS_ZSET, rmt_strlen(REDIS_REPLY_STATUS_ZSET)) == 0) { cunit->key_type = REDIS_ZSET; arg = array_push(&args); *arg = sdsnew("zrange"); arg = array_push(&args); *arg = sdsdup(cunit->key); arg = array_push(&args); *arg = sdsnew("0"); arg = array_push(&args); *arg = sdsnew("-1"); ret = redis_msg_append_command_full_safe(msg, &args); if (ret != RMT_OK) { log_error("ERROR: msg append multi bulk len failed."); goto error; } while (array_n(&args) > 0) { arg = array_pop(&args); sdsfree(*arg); } } else if (msg_cmp_str(resp, (const uint8_t*)REDIS_REPLY_STATUS_HASH, rmt_strlen(REDIS_REPLY_STATUS_HASH)) == 0) { cunit->key_type = REDIS_HASH; arg = array_push(&args); *arg = sdsnew("hgetall"); arg = array_push(&args); *arg = sdsdup(cunit->key); ret = redis_msg_append_command_full_safe(msg, &args); if (ret != RMT_OK) { log_error("ERROR: msg append multi bulk len failed."); goto error; } while (array_n(&args) > 0) { arg = array_pop(&args); sdsfree(*arg); } } else { log_error("ERROR: response key type from node[%s] is error: ", rnode->addr); goto error; } msg->ptr = cunit; msg->resp_check = check_response; ret = send_msg_to_all(cunit, msg); if (ret != RMT_OK) { log_error("ERROR: send msg to source and target group failed."); goto error; } cunit->state = CHECK_UNIT_STATE_GET_VALUE; goto next_step; } if (cunit->state == CHECK_UNIT_STATE_GET_VALUE) { ASSERT(cunit->key != NULL); ASSERT(cunit->key_type >= 0); ASSERT(cunit->result1 == NULL || cunit->result2 == NULL); if (cunit->key_type == REDIS_STRING) { if (resp->type != MSG_RSP_REDIS_BULK) { log_error("ERROR: the response type for %s from node[%s] is error: %s", rnode->addr, msg_type_string(r->type), msg_type_string(resp->type)); goto error; } } else if (cunit->key_type == REDIS_LIST) { } else if (cunit->key_type == REDIS_SET) { } else if (cunit->key_type == REDIS_ZSET) { } else if (cunit->key_type == REDIS_HASH) { } else { NOT_REACHED(); } if (cunit->result1 == NULL) { cunit->result1 = resp; resp = NULL; } else if (cunit->result2 == NULL) { cunit->result2 = resp; resp = NULL; } else { NOT_REACHED(); } if (cunit->result1 != NULL && cunit->result2 != NULL) { if (cunit->key_type == REDIS_SET) { uint32_t j; bulks1 = get_multi_bulk_array_from_mbuf_list(cunit->result1->data); bulks2 = get_multi_bulk_array_from_mbuf_list(cunit->result2->data); if (bulks1 == NULL || bulks2 == NULL) { log_error("ERROR: get multi bulk array from mbufs failed"); goto error; } if (array_n(bulks1) != array_n(bulks2)) { chdata->err_inconsistent_value_keys_count ++; rmt_safe_snprintf(extra_err, 50, ", value is inconsistent\0"); goto error; } array_sort(bulks1, string_binary_cmp); array_sort(bulks2, string_binary_cmp); for (j = 0; j < array_n(bulks1); j ++) { bulk1 = array_get(bulks1, j); bulk2 = array_get(bulks2, j); if (string_binary_cmp(bulk1, bulk2) != 0) { chdata->err_inconsistent_value_keys_count ++; rmt_safe_snprintf(extra_err, 50, ", value is inconsistent\0"); goto error; } } } else if (cunit->key_type == REDIS_HASH) { struct array *hash_datas1, *hash_datas2; uint32_t hash_len; uint32_t j; struct hash_data *hd1, *hd2; hash_datas1 = hash_datas2 = NULL; bulks1 = get_multi_bulk_array_from_mbuf_list(cunit->result1->data); bulks2 = get_multi_bulk_array_from_mbuf_list(cunit->result2->data); if (bulks1 == NULL || bulks2 == NULL) { log_error("ERROR: get multi bulk array from mbufs failed"); goto error; } if (array_n(bulks1)%2 != 0 || array_n(bulks2)%2 != 0) { log_error("ERROR: bad hash value"); goto error; } if (array_n(bulks1) != array_n(bulks2)) { chdata->err_inconsistent_value_keys_count ++; rmt_safe_snprintf(extra_err, 50, ", value is inconsistent\0"); goto error; } hash_len = array_n(bulks1)/2; hash_datas1 = array_create(hash_len, sizeof(struct hash_data)); hash_datas2 = array_create(hash_len, sizeof(struct hash_data)); for (j = 0; j < hash_len; j ++) { hd1 = array_push(hash_datas1); hd2 = array_push(hash_datas2); bulk1 = array_pop(bulks1); bulk2 = array_pop(bulks1); hd1->field = *bulk1; hd1->value = *bulk2; bulk1 = array_pop(bulks2); bulk2 = array_pop(bulks2); hd2->field = *bulk1; hd2->value = *bulk2; } array_sort(hash_datas1, hash_data_field_cmp); array_sort(hash_datas2, hash_data_field_cmp); for (j = 0; j < array_n(bulks1); j ++) { hd1 = array_get(hash_datas1, j); hd2 = array_get(hash_datas2, j); if (string_binary_cmp(hd1->field, hd2->field) != 0) { chdata->err_inconsistent_value_keys_count ++; rmt_safe_snprintf(extra_err, 50, ", value is inconsistent\0"); if (hash_datas1 != NULL) { while (array_n(hash_datas1) > 0) { hd1 = array_pop(hash_datas1); sdsfree(hd1->field); sdsfree(hd1->value); } array_destroy(hash_datas1); hash_datas1 = NULL; } if (hash_datas2 != NULL) { while (array_n(hash_datas2) > 0) { hd2 = array_pop(hash_datas2); sdsfree(hd2->field); sdsfree(hd2->value); } array_destroy(hash_datas2); hash_datas2 = NULL; } goto error; } if (string_binary_cmp(hd1->value, hd2->value) != 0) { chdata->err_inconsistent_value_keys_count ++; rmt_safe_snprintf(extra_err, 50, ", value is inconsistent\0"); if (hash_datas1 != NULL) { while (array_n(hash_datas1) > 0) { hd1 = array_pop(hash_datas1); sdsfree(hd1->field); sdsfree(hd1->value); } array_destroy(hash_datas1); hash_datas1 = NULL; } if (hash_datas2 != NULL) { while (array_n(hash_datas2) > 0) { hd2 = array_pop(hash_datas2); sdsfree(hd2->field); sdsfree(hd2->value); } array_destroy(hash_datas2); hash_datas2 = NULL; } goto error; } } if (hash_datas1 != NULL) { while (array_n(hash_datas1) > 0) { hd1 = array_pop(hash_datas1); sdsfree(hd1->field); sdsfree(hd1->value); } array_destroy(hash_datas1); hash_datas1 = NULL; } if (hash_datas2 != NULL) { while (array_n(hash_datas2) > 0) { hd2 = array_pop(hash_datas2); sdsfree(hd2->field); sdsfree(hd2->value); } array_destroy(hash_datas2); hash_datas2 = NULL; } } else if (msg_data_compare(cunit->result1, cunit->result2) != 0) { chdata->err_inconsistent_value_keys_count ++; rmt_safe_snprintf(extra_err, 50, ", value is inconsistent\0"); goto error; } msg_put(cunit->result1); msg_free(cunit->result1); cunit->result1 = NULL; msg_put(cunit->result2); msg_free(cunit->result2); cunit->result2 = NULL; if (bulks1 != NULL) { while (array_n(bulks1) > 0) { bulk1 = array_pop(bulks1); sdsfree(*bulk1); } array_destroy(bulks1); bulks1 = NULL; } if (bulks2 != NULL) { while (array_n(bulks2) > 0) { bulk2 = array_pop(bulks2); sdsfree(*bulk2); } array_destroy(bulks2); bulks2 = NULL; } msg = msg_get(r->mb, 1, REDIS_DATA_TYPE_CMD); if (msg == NULL) { log_error("ERROR: out of memory."); goto error; } arg = array_push(&args); *arg = sdsnew("ttl"); arg = array_push(&args); *arg = sdsdup(cunit->key); ret = redis_msg_append_command_full_safe(msg, &args); if (ret != RMT_OK) { log_error("ERROR: msg append multi bulk len failed."); goto error; } while (array_n(&args) > 0) { arg = array_pop(&args); sdsfree(*arg); } msg->ptr = cunit; msg->resp_check = check_response; ret = send_msg_to_all(cunit, msg); if (ret != RMT_OK) { log_error("ERROR: send msg to source and target group failed."); goto error; } cunit->state = CHECK_UNIT_STATE_GET_EXPIRE; } goto next_step; } if (cunit->state == CHECK_UNIT_STATE_GET_EXPIRE) { ASSERT(cunit->key != NULL); ASSERT(cunit->key_type >= 0); ASSERT(cunit->result1 == NULL || cunit->result2 == NULL); if (resp->type != MSG_RSP_REDIS_INTEGER) { log_error("ERROR: the response type for command 'ttl' from node[%s] is error: %s", rnode->addr, msg_type_string(resp->type)); goto error; } if (cunit->result1 == NULL) { cunit->result1 = resp; resp = NULL; } else if (cunit->result2 == NULL) { cunit->result2 = resp; resp = NULL; } else { NOT_REACHED(); } if (cunit->result1 != NULL && cunit->result2 != NULL) { if (msg_data_compare(cunit->result1, cunit->result2) != 0) { int mistake = (int)cunit->result1->integer - (int)cunit->result2->integer; ASSERT(mistake != 0); if (abs(mistake) > TTL_MISTAKE_CAN_BE_ACCEPT) { chdata->err_inconsistent_expire_keys_count ++; rmt_safe_snprintf(extra_err, 50, ", remaining time are %"PRIu32" and %"PRIu32"\0", cunit->result1->integer, cunit->result2->integer); goto error; } } /* OK, this key is consistent between source group and target group */ goto done; } goto next_step; } done: check_unit_destroy(cunit); next_step: msg_put(r); msg_free(r); if (resp != NULL) { msg_put(resp); msg_free(resp); } array_deinit(&args); return RMT_OK; error: chdata->err_check_keys_count ++; if (cunit->key != NULL) { log_error("ERROR: key checked failed: %s%s. key(len:%zu, type:%s): %.*s", get_check_error(cunit), extra_err, sdslen(cunit->key), get_redis_type_string(cunit->key_type), sdslen(cunit->key), cunit->key); } else { log_error("ERROR: key checked failed: %s%s.", get_check_error(cunit), extra_err); } MSG_DUMP(r, LOG_ERR, 1); msg_put(r); msg_free(r); if (resp != NULL) { MSG_DUMP(resp, LOG_ERR, 1); msg_put(resp); msg_free(resp); } if (msg != NULL) { msg_put(msg); msg_free(msg); } check_unit_destroy(cunit); while (array_n(&args) > 0) { arg = array_pop(&args); sdsfree(*arg); } array_deinit(&args); if (bulks1 != NULL) { while (array_n(bulks1) > 0) { bulk1 = array_pop(bulks1); sdsfree(*bulk1); } array_destroy(bulks1); bulks1 = NULL; } if (bulks2 != NULL) { while (array_n(bulks2) > 0) { bulk2 = array_pop(bulks2); sdsfree(*bulk2); } array_destroy(bulks2); bulks2 = NULL; } return RMT_OK; }
static void check_begin(aeEventLoop *el, int fd, void *privdata, int mask) { int ret; redis_node *srnode = privdata; thread_data *cdata = srnode->write_data; mbuf_base *mb = cdata->srgroup->mb; struct msg *msg; check_unit *cunit; RMT_NOTUSED(el); RMT_NOTUSED(fd); RMT_NOTUSED(privdata); RMT_NOTUSED(mask); ASSERT(el == cdata->loop); ASSERT(fd == srnode->sk_event); if (cdata->sent_keys_count - cdata->finished_keys_count > MAX_UNITS_HOLD_PER_THREAD) { return; } if (cdata->sent_keys_count >= cdata->keys_count) { aeDeleteFileEvent(cdata->loop,srnode->sk_event,AE_WRITABLE); return; } cunit = check_unit_create(); if (cunit == NULL) { log_error("Error: out of memory."); goto error; } cunit->srnode = srnode; cunit->cdata = cdata; cunit->state = CHECK_UNIT_STATE_GET_KEY; cdata->sent_keys_count ++; msg = msg_get(mb, 1, REDIS_DATA_TYPE_CMD); if (msg == NULL) { log_error("ERROR: msg get failed."); goto error; } ret = msg_append_full(msg, (uint8_t *)REDIS_RANDOMKEY, strlen(REDIS_RANDOMKEY)); if (ret != RMT_OK) { log_error("ERROR: msg append REDIS_RANDOMKEY failed."); goto error; } msg->ptr = cunit; msg->resp_check = check_response; /*send msg to source msg*/ ret = prepare_send_msg(srnode, msg, srnode); if (ret != RMT_OK) { goto error; } return; error: aeDeleteFileEvent(cdata->loop,srnode->sk_event,AE_READABLE|AE_WRITABLE); aeStop(cdata->loop); }
static int send_msg_to_all(check_unit *cunit, struct msg *msg) { int ret; thread_data *cdata = cunit->cdata; redis_group *trgroup = cdata->trgroup; redis_node *trnode; struct msg *msg_same; listNode *lnode; struct mbuf *mbuf; if (cunit == NULL || msg == NULL) { return RMT_ERROR; } msg_same = msg_get(msg->mb, msg->request, msg->kind); if (msg_same == NULL) { log_error("ERROR: msg clone failed."); msg_put(msg); msg_free(msg); msg = NULL; return RMT_ERROR; } lnode = listFirst(msg->data); while (lnode) { mbuf = listNodeValue(lnode); lnode = lnode->next; ret = msg_append_full(msg_same, mbuf->pos, mbuf_length(mbuf)); if (ret != RMT_OK) { log_error("ERROR: out of memory."); msg_put(msg_same); msg_free(msg_same); msg = NULL; return RMT_ERROR; } } msg_same->ptr = msg->ptr; msg_same->resp_check = msg->resp_check; ret = prepare_send_msg(cunit->srnode, msg, cunit->srnode); if (ret != RMT_OK) { msg_put(msg); msg_free(msg); msg = NULL; msg_put(msg_same); msg_free(msg_same); return RMT_ERROR; } msg = NULL; trnode = trgroup->get_backend_node(trgroup, (uint8_t *)cunit->key, (uint32_t)sdslen(cunit->key)); if(prepare_send_msg(trnode, msg_same, trnode) != RMT_OK){ msg_put(msg_same); msg_free(msg_same); return RMT_ERROR; } return RMT_OK; }