int init_optimize() {
if (tdhs_optimize_bloom_filter_group > 0
&& tdhs_optimize_bloom_filter_num_buckets > 0) {
filters =
(tdhs_bloom_filter_t*) TAOBAO_MALLOC(sizeof(tdhs_bloom_filter_t)*tdhs_optimize_bloom_filter_group);
if (filters == NULL) {
easy_error_log(
"TDHS: don't have enough memory for tdhs_bloom_filter_t");
return EASY_ERROR;
}
memset(filters, 0,
sizeof(tdhs_bloom_filter_t) * tdhs_optimize_bloom_filter_group);
for (size_t i = 0; i < tdhs_optimize_bloom_filter_group; i++) {
filters[i].filter = create_bfilter(
tdhs_optimize_bloom_filter_num_buckets);
if (filters[i].filter == NULL) {
easy_error_log(
"TDHS: don't have enough memory for bloom_filter");
//free already alloced filter
for (size_t j = 0; j < i; j++) {
filters[j].destory();
}
TAOBAO_FREE(filters);
filters = NULL;
return EASY_ERROR;
}
}
}
return EASY_OK;
}
/**
* 程序入口
*/
int main(int argc, char** argv) {
cmdline_param cp;
easy_listen_t* listen;
easy_io_handler_pt io_handler;
int ret;
// default
memset(&cp, 0, sizeof(cmdline_param));
cp.io_thread_cnt = 1;
// parse cmd line
if (parse_cmd_line(argc, argv, &cp) == EASY_ERROR)
return EASY_ERROR;
// 检查必需参数
if (cp.port == 0) {
print_usage(argv[0]);
return EASY_ERROR;
}
// 对easy_io初始化, 设置io的线程数, file的线程数
if (!easy_io_create(cp.io_thread_cnt)) {
easy_error_log("easy_io_init error.\n");
return EASY_ERROR;
}
// 为监听端口设置处理函数，并增加一个监听端口
easy_io_var.tcp_defer_accept = 0;
memset(&io_handler, 0, sizeof(io_handler));
io_handler.decode = echo_decode;
io_handler.encode = echo_encode;
io_handler.process = echo_process;
io_handler.on_connect = echo_connect;
if ((listen = easy_io_add_listen(NULL, cp.port, &io_handler)) == NULL) {
easy_error_log("easy_io_add_listen error, port: %d, %s\n",
cp.port, strerror(errno));
return EASY_ERROR;
} else {
easy_error_log("listen start, port = %d\n", cp.port);
}
// 起处理速度统计定时器
ev_timer stat_watcher;
easy_io_stat_t iostat;
easy_io_stat_watcher_start(&stat_watcher, 5.0, &iostat, NULL);
// 起线程并开始
if (easy_io_start()) {
easy_error_log("easy_io_start error.\n");
return EASY_ERROR;
}
// 等待线程退出
ret = easy_io_wait();
easy_io_destroy();
return ret;
}
//only use for bulk request
void tdhs_dbcontext::lock_table() {
tb_assert(lock==NULL);
TABLE** tables;
bool is_malloc = false;
if (!bulk) {
return;
}
if (opened_table_num <= DBCONTEXT_MAX_CACHE_LOCK_TABLES) {
tables = need_lock_tables;
} else {
tables = (TABLE**) TAOBAO_MALLOC(sizeof(TABLE*)*opened_table_num);
if (tables == NULL) {
easy_error_log("TDHS: lock_table failed for not enough memory!");
return;
}
is_malloc = true;
opened_table_t * t;
uint32_t i = 0;
uint32_t j = 0;
easy_hash_list_t * node;
easy_hash_for_each(i,node,hash_table_for_opened) {
t =
(opened_table_t*) ((char*) node
- hash_table_for_opened->offset);
tables[j++] = (TABLE*) t->mysql_table;
}
tb_assert(opened_table_num==j);
}
static TDHS_INLINE int write_update_header_to_response(tdhs_packet_t& packet,
char type, size_t field_size, size_t num) {
uint32_t size = sizeof(uint32_t) + num;
uint32_t alloc_size = size + (sizeof(uint32_t) + field_size) * num;
easy_pool_t *pool = packet.pool;
tb_assert(pool!=NULL);
easy_buf_t *b = easy_buf_create(pool,
alloc_size + TDH_SOCKET_HEADER_LENGTH);
if (b == NULL) {
easy_error_log(
"TDHS:write_data_header_to_response create packet buf failed!");
return EASY_ERROR;
}
b->last += TDH_SOCKET_HEADER_LENGTH; //留出空间给header
packet.stream_buffer = b->pos;
packet.wbuff = b;
*((uint32_t*) (b->last)) = htonl(num); //放两个字段 update_row_num 和 change_row_num
b->last += sizeof(uint32_t);
for (size_t i = 0; i < num; i++) {
*((char*) b->last) = type;
b->last++;
}
packet.length += size;
return EASY_OK;
}
static TDHS_INLINE int write_data_header_to_response(tdhs_packet_t& packet,
uint32_t field_num, char* field_types) {
uint32_t size = sizeof(uint32_t) + field_num;
uint32_t alloc_size =
size > tdhs_write_buff_size ? size : tdhs_write_buff_size;
easy_pool_t *pool = packet.pool;
tb_assert(pool!=NULL);
easy_buf_t *b = easy_buf_create(pool,
alloc_size + TDH_SOCKET_HEADER_LENGTH);
if (b == NULL) {
easy_error_log(
"TDHS:write_data_header_to_response create packet buf failed!");
return EASY_ERROR;
}
b->last += TDH_SOCKET_HEADER_LENGTH; //留出空间给header
packet.stream_buffer = b->pos;
packet.wbuff = b;
*((uint32_t*) (b->last)) = htonl(field_num);
b->last += sizeof(uint32_t);
for (uint32_t i = 0; i < field_num; i++) {
*((char*) b->last) = field_types[i];
b->last++;
}
packet.length += size;
return EASY_OK;
}
tdhs_optimize_t optimize(const tdhs_request_t &request) {
if (request.type == REQUEST_TYPE_GET
|| request.type == REQUEST_TYPE_COUNT) {
switch (thread_strategy) {
case TDHS_THREAD_LV_1:
return TDHS_QUICK;
break;
case TDHS_THREAD_LV_3:
#ifdef TDHS_ROW_CACHE
if (tdhs_optimize_on) {
return TDHS_QUICK;
break;
}
#else
if (tdhs_optimize_on && filters) {
uint64_t hash = calc_hash_get(request);
request.optimize_hash = hash;
tdhs_bloom_filter_t &filter = filters[current_filter_index
% tdhs_optimize_bloom_filter_group];
if (filter.check(&hash)) {
optimize_lv3_assign_to_quick_count++;
return TDHS_QUICK;
} else {
optimize_lv3_assign_to_slow_count++;
return TDHS_SLOW;
}
break;
}
#endif
easy_info_log("TDHS: tdhs_optimize_on is off,use lv2");
default:
if (thread_strategy != TDHS_THREAD_LV_3) {
easy_error_log(
"TDHS: error thread_strategy [%d],default use lv2",
thread_strategy);
}
case TDHS_THREAD_LV_2:
uint64_t hash = easy_hash_key(poll_key++);
if (hash % (tdhs_thread_num + active_slow_read_thread_num)
<= tdhs_thread_num) {
return TDHS_QUICK;
} else {
return TDHS_SLOW;
}
break;
}
} else { //MARK 针对request type 需要进行判断
return TDHS_WRITE;
}
return TDHS_QUICK;
}
static TDHS_INLINE int tdhs_response_batch(tdhs_packet_t *packet) {
packet->command_id_or_response_code = CLIENT_STATUS_MULTI_STATUS;
packet->length = 0; //这个包的length是0
easy_pool_t *pool = packet->pool;
tb_assert(pool!=NULL);
easy_buf_t *b = easy_buf_create(pool, TDH_SOCKET_HEADER_LENGTH);
if (b == NULL) {
easy_error_log("TDHS:tdhs_response_batch create packet buf failed!");
return EASY_ERROR;
}
b->last += TDH_SOCKET_HEADER_LENGTH; //留出空间给header
packet->wbuff = b;
return EASY_OK;
}
static TDHS_INLINE int decode_to_batch(tdhs_request_t & req,
tdhs_packet_t& packet) {
tdhs_packet_t *batch_packet = packet.next;
while (batch_packet) {
if (decode_request_by_binary_v2(batch_packet->req,
*batch_packet) != EASY_OK) {
return EASY_ERROR;
}
if (batch_packet->req.type == REQUEST_TYPE_GET
|| batch_packet->req.type == REQUEST_TYPE_COUNT) {
easy_error_log("TDHS:batch request can't include read request!");
return EASY_ERROR;
}
batch_packet = batch_packet->next;
}
return EASY_OK;
}
static TDHS_INLINE int tdhs_response_error(tdhs_packet_t *packet,
tdhs_client_status_t status, uint32_t code) {
uint32_t length = sizeof(uint32_t);
packet->command_id_or_response_code = status;
packet->length = length;
easy_pool_t *pool = packet->pool;
tb_assert(pool!=NULL);
easy_buf_t *b = easy_buf_create(pool, length + TDH_SOCKET_HEADER_LENGTH);
if (b == NULL) {
easy_error_log("TDHS:tdhs_response_error create packet buf failed!");
return EASY_ERROR;
}
b->last += TDH_SOCKET_HEADER_LENGTH; //留出空间给header
packet->wbuff = b;
*((uint32_t*) (b->last)) = htonl(code);
b->last += length;
return EASY_OK;
}
void tdhs_dbcontext::open_table(tdhs_request_t &req) {
//MARK 针对request type 需要进行判断,不需要open table的直接退出
if (req.type == REQUEST_TYPE_BATCH) {
return;
}
if (pool_for_open_table == NULL) {
pool_for_open_table = easy_pool_create(MAX_POOL_SIZE_FOR_OPEN_TABLE);
}
if (already_cached_table_num > 0) {
tb_assert(already_cached_table_num<=cache_table_num);
for (unsigned int i = 0; i < already_cached_table_num; i++) {
cached_table_t &cached_t = cached_table[i];
if (cached_t.table != NULL) {
TABLE* t = cached_t.table;
opened_table_t& opened_t = for_cached_opened_table[i];
opened_t.mysql_table = t;
easy_hash_add(hash_table_for_opened,
cached_t.hash_code_for_table > 0 ?
cached_t.hash_code_for_table :
make_hash_code_for_table(t->s->db.str,t->s->db.length,t->s->table_name.str,t->s->table_name.length)
, &opened_t.hash);
if (opened_table_num < DBCONTEXT_MAX_CACHE_LOCK_TABLES) {
need_lock_tables[opened_table_num++] = t;
} else {
opened_table_num++;
}
cached_t.table = NULL; //置空
cached_t.hash_code_for_table = 0; //置空
}
}
already_cached_table_num = 0; //置0
}
{
opened_table_t * opened_table = (opened_table_t *) easy_hash_find_ex(
hash_table_for_opened, req.table_info.hash_code_table(),
compare_table_info, &req.table_info);
if (opened_table != NULL) {
req.opened_table = opened_table;
} else {
opened_table = (opened_table_t *) easy_pool_calloc(
pool_for_open_table, sizeof(opened_table_t));
if (opened_table != NULL) {
req.opened_table = opened_table;
TABLE* t = tdhs_open_table(thd, req.table_info,
this->need_write());
if (t != NULL) {
req.opened_table->mysql_table = t;
easy_hash_add(hash_table_for_opened,
req.table_info.hash_code_table(),
&req.opened_table->hash);
if (opened_table_num < DBCONTEXT_MAX_CACHE_LOCK_TABLES) {
need_lock_tables[opened_table_num++] = t;
} else {
opened_table_num++;
}
}
} else {
//if opened_table==NULL because of memory is not enough
easy_error_log(
"TDHS:not enough memory for calloc opened_table_t!");
}
}
}
}
void*tdhs_decode(easy_message_t *m) {
tdhs_packet_t *packet;
uint32_t len, magic_code, com_or_resp, seq_id, reverse, datalen;
uint32_t batch_request_num = 0;
unsigned long long int now;
size_t i;
char *pos; //解析batch请求用
uint32_t left_length; //判断batch请求长度用
if ((len = m->input->last - m->input->pos) < TDH_SOCKET_HEADER_LENGTH)
return NULL;
// packet
magic_code = ntohl(*((uint32_t *) (m->input->pos)));
if (magic_code != TDHS_MAGIC_CODE) {
easy_error_log("TDHS:magic_code is invalid: %d\n", magic_code);
m->status = EASY_ERROR;
return NULL;
}
GET_HEADER_INFO(com_or_resp, seq_id, reverse, datalen, m->input->pos)
batch_request_num = com_or_resp == REQUEST_TYPE_BATCH ? reverse : 0;
if (datalen > 0x4000000) { // 64M
easy_error_log("TDHS:data_len is invalid: %d\n", datalen);
m->status = EASY_ERROR;
return NULL;
}
// 长度不够
len -= TDH_SOCKET_HEADER_LENGTH;
if (len < datalen) {
m->next_read_len = datalen - len;
return NULL;
}
// alloc packet
if ((packet = (tdhs_packet_t *) easy_pool_calloc(m->pool,
sizeof(tdhs_packet_t) * (1 + batch_request_num))) == NULL) {
m->status = EASY_ERROR;
return NULL;
}
now = tdhs_micro_time();
packet->command_id_or_response_code = com_or_resp;
packet->seq_id = seq_id;
packet->reserved = reverse;
packet->length = datalen;
m->input->pos += TDH_SOCKET_HEADER_LENGTH;
packet->rdata = (char *) m->input->pos;
packet->pool = m->pool; //此处设置使用message的pool
packet->start_time = now;
//处理batch请求
pos = packet->rdata;
left_length = packet->length;
for (i = 0; i < batch_request_num; i++) {
tdhs_packet_t *batch_packet = packet + 1 + i;
if (left_length < TDH_SOCKET_HEADER_LENGTH) {
//长度不全导致没法完整解析batch请求的头
easy_error_log( "TDHS:batch left_length is invalid: %d %d\n",
left_length);
m->status = EASY_ERROR;
return NULL;
}
GET_HEADER_INFO(com_or_resp, seq_id, reverse, datalen, pos)
pos += TDH_SOCKET_HEADER_LENGTH;
left_length -= TDH_SOCKET_HEADER_LENGTH;
if (left_length < datalen) {
//长度不全导致没法完整解析batch请求
easy_error_log(
"TDHS:batch data_len is invalid: %d ,the left_length is %d\n",
datalen, left_length);
m->status = EASY_ERROR;
return NULL;
}
batch_packet->command_id_or_response_code = com_or_resp;
batch_packet->seq_id = seq_id;
//batch 目前不支持嵌套 ,所以不设置reserved
batch_packet->length = datalen;
batch_packet->rdata = pos;
batch_packet->pool = m->pool; //此处设置使用message的pool
batch_packet->start_time = now;
(batch_packet - 1)->next = batch_packet; //形成链
pos += datalen;
left_length -= datalen;
}
m->input->pos += packet->length;
return packet;
}
