spx_private bool_t ydb_storage_get_remote(struct ydb_storage_configurtion *c,\ struct spx_job_context_transport *arg){/*{{{*/ err_t err = 0; struct spx_vector_iter *iter = spx_vector_iter_init(c->trackers ,&err); if(NULL == iter){ SpxLog2(c->log,SpxLogError,err,\ "init the trackers iter is fail."); return false; } struct ydb_tracker *t = NULL; while(NULL != (t = spx_vector_iter_next(iter))){ if(NULL == t->sjc){ t->sjc = (struct spx_job_context *) spx_job_context_new(0,arg,&err); if(NULL == t->sjc){ SpxLog2(c->log,SpxLogError,err,"alloc heartbeat nio context is fail."); continue; } } err = ydb_storage_query_remote(c->log,t,c->groupname,\ c->machineid,c->syncgroup); if(0 != err){ t->sjc->err = err; SpxLogFmt2(t->sjc->log,SpxLogError,err,\ "query sync storage to tracker %s:%d is fail.",\ t->host.ip,t->host.port); } } spx_vector_iter_free(&iter); return true; }/*}}}*/
spx_private void *spx_nio_thread_context_new(size_t idx,void *arg,err_t *err){ struct spx_thread_context_node *tcn = (struct spx_thread_context_node *)arg; struct spx_nio_thread_context *context = spx_alloc_alone(sizeof(*context),err); if(NULL == context){ SpxLog2(tcn->log,SpxLogError,*err,\ "alloc nio thread context is fail."); return NULL; } context->loop = ev_loop_new(EVFLAG_AUTO); context->log = tcn->log; context->idx = idx; context->thread_notify_handle = tcn->thread_notify; if(-1 == pipe(context->pipe)){ SpxLog2(tcn->log,SpxLogError,*err,\ "open the nio thread pips is fail."); *err = errno; SpxFree(context); return NULL; } if((0 != (*err = spx_set_nb(context->pipe[0]))) \ ||(0 != (*err = spx_set_nb(context->pipe[0])))){ SpxLog2(tcn->log,SpxLogError,*err,\ "set pipe noblacking is fail."); SpxFree(context); return NULL; } return context; }
spx_private void ydb_storage_syncquery_nio_body_reader(int fd, struct spx_job_context *jc){/*{{{*/ spx_nio_reader_body_handler(fd,jc); if(ENOENT == jc->err){ SpxLog1(jc->log,SpxLogWarn,\ "not find out the sync storages."); goto r1; } if(0 != jc->err){ SpxLog2(jc->log,SpxLogError,jc->err,\ "recv the regedit response is fail."); goto r1; } struct spx_msg *ctx = jc->reader_body_ctx; if(NULL == ctx){ SpxLog2(jc->log,SpxLogError,jc->err,\ "no recved the body ctx."); goto r1; } struct ydb_storage_slave *s = NULL; size_t unit_size = YDB_MACHINEID_LEN + SpxIpv4Size + SpxI32Size + SpxI32Size; size_t numbs = jc->reader_header->bodylen / unit_size; size_t i = 0; for( ; i < numbs; i++){ string_t machineid = spx_msg_unpack_string(ctx,YDB_MACHINEID_LEN,&(jc->err)); string_t ip = spx_msg_unpack_string(ctx,SpxIpv4Size,&(jc->err)); i32_t port = spx_msg_unpack_i32(ctx); u32_t status = spx_msg_unpack_u32(ctx); spx_map_get(g_ydb_storage_slaves,machineid,spx_string_len(machineid),\ (void **) &s,NULL); if(NULL == s){ s = spx_alloc_alone(sizeof(*s),&(jc->err)); if(NULL == s){ continue; } s->state = status; s->machineid = machineid; s->host.port = port; s->host.ip = ip; spx_map_insert(g_ydb_storage_slaves,machineid,spx_string_len(machineid), s,sizeof(s)); } if(port != s->host.port){ s->host.port = port; } if(0 != spx_string_casecmp_string(machineid,s->machineid)){ spx_string_free(s->machineid); s->machineid = machineid; } s->state = status; } r1: spx_job_context_clear(jc); }/*}}}*/
spx_private err_t ydb_storage_query_remote(SpxLogDelegate *log, struct ydb_tracker *t,string_t groupname, string_t machineid,string_t syncgroup){/*{{{*/ err_t err = 0; struct spx_job_context *sjc = t->sjc; int fd = spx_socket_new(&err); if(0 >= fd){ SpxLog2(log,SpxLogError,err,"create query sync socket fd is fail."); return err; } if(0 != (err = spx_set_nb(fd))){ SpxLog2(log,SpxLogError,err,"set socket nonblacking is fail."); goto r1; } if(0 != (err = spx_socket_set(fd,SpxKeepAlive,SpxAliveTimeout,\ SpxDetectTimes,SpxDetectTimeout,\ SpxLinger,SpxLingerTimeout,\ SpxNodelay,\ true,30))){ SpxLog2(log,SpxLogError,err,"set socket operator is fail."); goto r1; } if(0 != (err = spx_socket_connect(fd,t->host.ip,t->host.port))){ SpxLogFmt2(log,SpxLogError,err,\ "conntect to tracker:%s:%d is fail.",\ t->host.ip,t->host.port); goto r1; } struct spx_msg_header *writer_header = NULL; writer_header = spx_alloc_alone(sizeof(*writer_header),&err); if(NULL == writer_header){ SpxLog2(log,SpxLogError,err,\ "alloc writer header is fail."); goto r1; } sjc->writer_header = writer_header; writer_header->version = YDB_VERSION; writer_header->protocol = YDB_QUERY_SYNC_STORAGES; writer_header->bodylen = YDB_GROUPNAME_LEN + YDB_MACHINEID_LEN \ + YDB_SYNCGROUP_LEN ; struct spx_msg *ctx = spx_msg_new(writer_header->bodylen,&err); if(NULL == ctx){ SpxLog2(log,SpxLogError,err,\ "alloc writer body is fail."); goto r1; } sjc->writer_body_ctx = ctx; spx_msg_pack_fixed_string(ctx,groupname,YDB_GROUPNAME_LEN); spx_msg_pack_fixed_string(ctx,machineid,YDB_MACHINEID_LEN); spx_msg_pack_fixed_string(ctx,syncgroup,YDB_SYNCGROUP_LEN); sjc->fd = fd; spx_nio_regedit_writer(sloop,sjc->fd,sjc); return err; r1: SpxClose(fd); return err; }/*}}}*/
struct spx_msg_header *spx_read_header_nb(SpxLogDelegate *log,int fd,err_t *err){/*{{{*/ size_t len = 0; struct spx_msg *hbuff = spx_msg_new(SpxMsgHeaderSize,err); if(NULL == hbuff){ SpxLog2(log,SpxLogError,*err, "new msg-ctx for header is fail."); return NULL; } *err = spx_read_to_msg_nb(fd,hbuff,SpxMsgHeaderSize,&len); if(0 != *err){ SpxLogFmt2(log,SpxLogError,*err, "recv header buff:%d,realsize:%d.", SpxMsgHeaderSize,len); SpxMsgFree(hbuff); return NULL; } struct spx_msg_header *h = spx_msg_to_header(hbuff,err); if(NULL == h){ SpxLog2(log,SpxLogError,*err, "convert msg-ctx to header is fail."); SpxMsgFree(hbuff); return NULL; } return h; }/*}}}*/
spx_private void ReciveRequest(EV_P_ ev_async *watcher, int revents){/*{{{*/ ev_async_stop(loop, watcher); struct server_context * ctx = (struct server_context *) watcher->data; if(NULL == ctx){ SpxLog1(g_log, SpxLogError, "ReciveRequest ctx is NULL"); return; } if(READ_HEADER == ctx->life_cycle){ char buf[LOG_HEADER_SIZE] = {0}; size_t len = 0; err_t err = ReciveRequest_GetRequest_ReadRequest(ctx->fd, buf, &len, LOG_HEADER_SIZE); if(0 == err){ log_header_unpack(buf, ctx->header); char * request = (char *) calloc(1, sizeof(char)*ctx->header->req_size); if(NULL == request){ SpxLog2(g_log, SpxLogError, err,"calloc log_header failed"); return; } ctx->req_size = ctx->header->req_size; ctx->request = request; ctx->life_cycle = READ_REQUEST; } else{ if(EAGAIN == err || EWOULDBLOCK == err || EINTR == err) { ev_once(loop, ctx->fd, EV_READ, ctx->timeout, ReciveRequest_GetRequest, ctx); return; } else{ SpxLog2(g_log, SpxLogError, err,"Read header failed"); CloseCTX(ctx); return; } } } if(READ_REQUEST == ctx->life_cycle){ printf("---------------ReadRequest-------------\n"); printf("req_size:%d\n", (int)ctx->req_size); err_t err = ReciveRequest_GetRequest_ReadRequest(ctx->fd, ctx->request, &ctx->req_len, ctx->req_size); printf("read request complete\n" ); if(0 == err){ ctx->life_cycle = PARSE_REQUEST; RegisterAayncWatcher(&ctx->async_watcher, ParserRequest, ctx); ev_async_start(loop, &ctx->async_watcher); ev_async_send(loop, &ctx->async_watcher); return; }else{ if(EAGAIN == err || EWOULDBLOCK == err || EINTR == err) { ev_once(loop, ctx->fd, EV_READ, ctx->timeout, ReciveRequest_GetRequest, ctx); return; }else{ SpxLog2(g_log, SpxLogError, err,"ReciveRequest_GetRequest_ReadRequest Failed"); CloseCTX(ctx); return; } } } }/*}}}*/
struct spx_list *spx_nio_threadpool_create(\ SpxLogDelegate *log,\ u32_t threadsize,\ size_t stack_size,\ SpxThreadNotifyDelegate *thread_notify_handle,\ err_t *err){ struct spx_thread_context_node tcn; SpxZero(tcn); tcn.thread_notify = thread_notify_handle; tcn.log = log; struct spx_list *nio_thread_contexts = spx_list_init(log,threadsize,\ spx_nio_thread_context_new,&tcn,\ spx_nio_thread_context_free,\ err); if(0 == threadsize){ SpxLog1(log,SpxLogError,\ "the argument:threadsize is 0"); return NULL; } if(NULL == nio_thread_contexts){ SpxLog2(log,SpxLogError,*err,\ "alloc nio thread context is fail."); return NULL; } pthread_attr_t attr; pthread_attr_init(&attr); size_t ostack_size = 0; pthread_attr_getstacksize(&attr, &ostack_size); do{ if (ostack_size != stack_size && (0 != (*err = pthread_attr_setstacksize(&attr,stack_size)))){ SpxLog2(log,SpxLogError,*err,\ "set thread stack size is fail."); goto r1; } u32_t i = 0; for( ; i < threadsize; i++){ struct spx_nio_thread_context * n = spx_list_get(nio_thread_contexts,i); n->log = log; if (0 !=(*err = pthread_create(&(n->tid), &attr, spx_nio_thread_listen, n))){ SpxLog2(log,SpxLogError,*err,\ "create nio thread is fail."); goto r1; } } }while(false); pthread_attr_destroy(&attr); return nio_thread_contexts; r1: pthread_attr_destroy(&attr); spx_list_free(&nio_thread_contexts); return NULL; }
spx_private void ReciveRequest(EV_P_ ev_async *watcher, int revents){/*{{{*/ ev_async_stop(loop, watcher); struct server_context * ctx = (struct server_context *) watcher->data; if(NULL == ctx){ SpxLog1(g_log, SpxLogError, "ReciveRequest ctx is NULL"); return; } err_t err = ReciveRequest_GetRequest_ReadRequest(ctx->fd, ctx->request, &ctx->req_len); if(0 == err){ RegisterAayncWatcher(&ctx->async_watcher, ParserRequest, ctx); ev_async_start(loop, &ctx->async_watcher); ev_async_send(loop, &ctx->async_watcher); printf("buf:%s\n",ctx->request); return; }else{ if(EAGAIN == err || EWOULDBLOCK == err || EINTR == err) { ev_once(loop, ctx->fd, EV_READ, ctx->timeout, ReciveRequest_GetRequest, ctx); return; }else{ SpxLog2(g_log, SpxLogError, err,"ReciveRequest_GetRequest_ReadRequest Failed"); if( -1 == err) CloseCTX(ctx); else RequestException(ctx, bad_request); return; } } }/*}}}*/
spx_private err_t CTXInit(struct server_context * ctx, long (*ServerHandler)(size_t req_size, char *request, char **response), struct server_config *conf){/*{{{*/ err_t err = 0; if(NULL == ctx){ SpxLog1(g_log, SpxLogError, "CTXInit ctx is NULL"); return errno; } ctx->p_nextCTX = NULL; ctx->p_preCTX = NULL; ctx->fd = 0; ctx->timeout = conf->timeout; ctx->life_cycle = 0; ctx->req_retry = 0; ctx->header_retry = 0; ctx->req_len = 0; ctx->req_size = 0; ctx->request = NULL; ctx->resp_retry = 0; ctx->resp_len = 0; ctx->split_size = 0; ctx->response = NULL; ctx->ServerHandler = ServerHandler; ctx->header = (struct log_header *) calloc(1, sizeof(struct log_header)); if(NULL== ctx->header){ SpxLog2(g_log, SpxLogError, err, "calloc struct log_header failed"); return err; } return err; }/*}}}*/
spx_private void Sender(EV_P_ ev_async *watcher, int revents){/*{{{*/ ev_async_stop(loop, watcher); struct server_context * ctx = (struct server_context *) watcher->data; if(NULL == ctx){ SpxLog1(g_log, SpxLogError, "Sender ctx is NULL"); return; } err_t err = Sender_WriteResponse(ctx->fd, ctx->response, &ctx->resp_len, &ctx->split_size); if((0 == err)&&(0 == ctx->split_size)){ if(ctx->resp_size == ctx->resp_len){ RequestFinish(ctx); }else{ int remain_size = ctx->resp_size - ctx->resp_len; if(remain_size >= SPLIT_SIZE){ ctx->split_size = SPLIT_SIZE; }else{ ctx->split_size = remain_size; } RegisterAayncWatcher(&ctx->async_watcher, Sender, ctx); ev_async_start(loop, &ctx->async_watcher); ev_async_send(loop, &ctx->async_watcher); return; } }else{ if((EAGAIN == err || EWOULDBLOCK == err || EINTR == err)||(ctx->resp_size > 0)) { ev_once(main_socket_loop, ctx->fd, EV_READ, ctx->timeout, Sender_ReWriteResponse, ctx); return; }else{ SpxLog2(g_log, SpxLogError, err,"Sender Failed"); RequestException(ctx, bad_request); } } }/*}}}*/
void *logdb_tracker_config_before_handle(SpxLogDelegate *log,err_t *err){ struct logdb_tracker_configurtion *config = (struct logdb_tracker_configurtion *) \ spx_alloc_alone(sizeof(*config),err); if(NULL == config){ SpxLog2(log,SpxLogError,*err,\ "alloc the tracker config is fail."); return NULL; } config->log = log; config->port = 1404; config->timeout = 30; config->waitting = 10; config->trytimes = 3; config->logsize = 10 * SpxMB; config->loglevel = SpxLogInfo; config->balance = YDB_TRACKER_BALANCE_LOOP; config->heartbeat = 35; config->daemon = true; config->stacksize = 128 * SpxKB; config->network_module_thread_size = 8; // config->notifier_module_thread_size = 4; config->task_module_thread_size = 4; config->context_size = 64; return config; }
spx_private void * CreateMainSocket(void *arg){/*{{{*/ struct server_config * conf = (struct server_config *) arg; SpxLogDelegate *log = conf->log; err_t err = 0; main_socket_loop = ev_loop_new(0); if(NULL == main_socket_loop){ SpxLog2(log, SpxLogError, err, "create main socket loop is fail."); goto r1; } int mainsocket = spx_socket_new(&err); if(0 == mainsocket){ SpxLog2(log, SpxLogError, err, "create main socket is fail."); goto r1; } if( 0 != (err = spx_set_nb(mainsocket))){ SpxLog2(log, SpxLogError, err, "set main socket nonblock is fail."); } if( 0 != (err = spx_socket_start(mainsocket, conf->ip, conf->port,\ true, conf->timeout,\ 3, conf->timeout,\ false, 0,\ true,\ true, conf->timeout, 1024))){ SpxLog2(log, SpxLogError, err, "start main socket is fail."); goto r1; } SpxLogFmt1(log, SpxLogMark, "main socket fd: %d" "and accepting...", mainsocket); RegisterIOWatcher(&monitor_watcher, mainsocket, Reciver, EV_READ, log); ev_io_start(main_socket_loop, &monitor_watcher); ev_run(main_socket_loop, 0); r1: SpxClose(mainsocket); return NULL; }/*}}}*/
/* *block skiplist and split */ pthread_t spx_block_skp_thread_new(SpxLogDelegate *log, err_t *err) { /*{{{*/ pthread_t tid = 0; if (0 != (*err = pthread_create(&tid, NULL, (void *)spx_block_skp_handler, log))) { SpxLog2(log, SpxLogError, *err, "thread spx_block_skp_handler failed"); return 0; } return tid; }/*}}}*/
spx_private void Sender_ReWriteResponse(int revents, void *arg){/*{{{*/ struct server_context *ctx = (struct server_context *) arg; if(NULL == ctx){ SpxLog1(g_log, SpxLogError, "Sender ctx is NULL"); return; } if(EV_ERROR & revents){ SpxLog1(g_log, SpxLogError, "EV_ERROR"); return; } if(EV_TIMEOUT & revents){ if((ctx->resp_retry++) >= RETRY_TIMES){ RequestException(ctx, bad_request); SpxLog1(g_log, SpxLogError, "EV_TIMEOUT"); return; }else{ ev_once(main_socket_loop, ctx->fd, EV_WRITE, ctx->timeout, Sender_ReWriteResponse, ctx); return; } } if(EV_WRITE & revents){ err_t err = Sender_WriteResponse(ctx->fd, ctx->response, &ctx->resp_len, &ctx->split_size); if((0 == err)&&(0 == ctx->split_size)){ if(ctx->resp_size == ctx->resp_len){ RequestFinish(ctx); }else{ int remain_size = ctx->resp_size - ctx->resp_len; if(remain_size >= SPLIT_SIZE){ ctx->split_size = SPLIT_SIZE; }else{ ctx->split_size = remain_size; } RegisterAayncWatcher(&ctx->async_watcher, Sender, ctx); ev_async_start(main_socket_loop, &ctx->async_watcher); ev_async_send(main_socket_loop, &ctx->async_watcher); return; } }else{ if((EAGAIN == err || EWOULDBLOCK == err || EINTR == err)||(ctx->resp_size > 0)) { ev_once(main_socket_loop, ctx->fd, EV_READ, ctx->timeout, Sender_ReWriteResponse, ctx); return; }else{ SpxLog2(g_log, SpxLogError, err,"Sender Failed"); RequestException(ctx, bad_request); } } } }/*}}}*/
struct spx_thread_context *spx_thread_context_new_alone(SpxLogDelegate *log,err_t *err){ struct spx_thread_context *tc = (struct spx_thread_context *) \ spx_alloc_alone(sizeof(*tc),err); if(NULL == tc){ SpxLog2(log,SpxLogError,*err,\ "alloc thread context is fail."); return NULL; } // tc->loop = ev_loop_new(0); tc->log = log; return tc; }
err_t spx_write_context_nb(SpxLogDelegate *log,int fd,struct spx_msg_context *ctx){/*{{{*/ err_t err = 0; size_t len = 0; struct spx_msg *hctx = spx_header_to_msg(ctx->header,SpxMsgHeaderSize,&err); if(NULL == hctx){ SpxLog2(log,SpxLogError,err, "header convert to msg-ctx is fail."); return err; } err = spx_write_from_msg_nb(fd,hctx,SpxMsgHeaderSize,&len); if(0 != err) { SpxLogFmt2(log,SpxLogError,err, "write header size:%lld,realsize:%lld.", SpxMsgHeaderSize,len); goto r1; } len = 0; if(ctx->is_sendfile){ err = spx_write_from_msg_nb(fd,ctx->body,ctx->header->offset,&len); if(0 != err){ SpxLogFmt2(log,SpxLogError,err, "write leading-date of body buffer:%lld,realsize:%lld.", ctx->header->offset,len); goto r1; } len = 0; err = spx_sendfile(fd,ctx->sendfile_fd,ctx->sendfile_begin,ctx->sendfile_size,&len); if(0 != err ){ SpxLogFmt2(log,SpxLogError,err, "sendfile size:%lld,realsize:%lld.", ctx->sendfile_size,len); goto r1; } } else { err = spx_write_from_msg_nb(fd,ctx->body,ctx->header->bodylen,&len); if(0 != err ){ SpxLogFmt2(log,SpxLogError,err, "write body buffer:%lld,realsize:%lld.", ctx->header->bodylen,len); goto r1; } } r1: if(NULL != hctx){ SpxMsgFree(hctx); } return err; }/*}}}*/
spx_private void Reciver(struct ev_loop *loop, ev_io *watcher, int revents){/*{{{*/ ev_io_stop(loop, watcher); pthread_attr_t attr; pthread_attr_init(&attr); SpxLogDelegate *log = (SpxLogDelegate *) watcher->data; err_t err = 0; while(true){ struct sockaddr_in client_addr; pthread_t tid = 0; unsigned int socket_len = 0; int client_sock = 0; socket_len = sizeof(struct sockaddr_in); client_sock = accept(watcher->fd, (struct sockaddr *) &client_addr, &socket_len); if (0 > client_sock){ if( EWOULDBLOCK == errno || EAGAIN == errno){ break; } break; } if( 0 == client_sock){ break; } if( 0 != (err = spx_set_nb(client_sock))){ SpxLog2(log, SpxLogError, err, "set main client_socket nonblock is fail."); } printf("\n----------------CLIENT:%d START CTX:%d-----------------------\n", client_sock, GetCTXCount()); struct server_context * ctx = CTXPop(); printf("\n----------------CLIENT:%d POP CTX:%d-----------------------\n", client_sock, GetCTXCount()); if(NULL != ctx){ ctx->fd = client_sock; RegisterAayncWatcher(&ctx->async_watcher, ReciveRequest, ctx); ev_async_start(loop, &ctx->async_watcher); ev_async_send(loop, &ctx->async_watcher); } } ev_io_start(loop, watcher); }/*}}}*/
spx_private void ReciveRequest_GetRequest(int revents, void *arg){/*{{{*/ struct server_context *ctx = (struct server_context *) arg; if(NULL == ctx){ SpxLog1(g_log, SpxLogError, "ReciveRequest_GetRequest ctx is NULL"); return; } if(EV_ERROR & revents){ SpxLog1(g_log, SpxLogError, "EV_ERROR"); return; } if(EV_TIMEOUT & revents){ if((ctx->req_retry++) >= RETRY_TIMES){ RequestException(ctx, bad_request); SpxLog1(g_log, SpxLogError, "EV_TIMEOUT"); return; }else{ ev_once(main_socket_loop, ctx->fd, EV_READ, ctx->timeout, ReciveRequest_GetRequest, ctx); return; } } if(EV_READ & revents){ err_t err = ReciveRequest_GetRequest_ReadRequest(ctx->fd, ctx->request, &ctx->req_len); if(0 == err){ RegisterAayncWatcher(&ctx->async_watcher, ParserRequest, ctx); ev_async_start(main_socket_loop, &ctx->async_watcher); ev_async_send(main_socket_loop, &ctx->async_watcher); }else{ if((EAGAIN == err || EWOULDBLOCK == err || EINTR == err)) { ev_once(main_socket_loop, ctx->fd, EV_READ, ctx->timeout, ReciveRequest_GetRequest, ctx); return; }else{ SpxLog2(g_log, SpxLogError, err,"ReciveRequest_GetRequest_ReadRequest Failed"); if( -1 == err) CloseCTX(ctx); else RequestException(ctx, bad_request); return; } } } }/*}}}*/
struct spx_msg *spx_read_body_nb(SpxLogDelegate *log,int fd,size_t size,err_t *err){/*{{{*/ size_t len = 0; struct spx_msg *ctx = spx_msg_new(size,err); if(NULL == ctx){ SpxLog2(log,SpxLogError,*err, "new msg-ctx for header is fail."); return NULL; } *err = spx_read_to_msg_nb(fd,ctx,size,&len); if(0 != *err){ SpxLogFmt2(log,SpxLogError,*err, "read body buffer:%lld realsize:%lld.", size,len); SpxMsgFree(ctx); return NULL; } return ctx; }/*}}}*/
spx_private void *spx_receiver_new(size_t idx,void *arg,err_t *err){/*{{{*/ struct spx_recvive_context_transport *rct = (struct spx_recvive_context_transport *) arg; if(NULL == rct){ *err = EINVAL; return NULL; } struct spx_receive_context *t = (struct spx_receive_context *) \ spx_alloc_alone(sizeof(*t),err); if(NULL == t){ SpxLog2(rct->log,SpxLogError,*err,\ "alloc trigger context is fail."); return NULL; } t->log = rct->log; t->idx = idx; t->receive_handler = rct->recviver; return t; }/*}}}*/
spx_private void ydb_storage_syncquery_handler(struct ev_loop *loop,\ ev_timer *w,int revents){/*{{{*/ struct spx_job_context *jc = (struct spx_job_context *) w->data; struct ydb_storage_configurtion *c = (struct ydb_storage_configurtion *) \ jc->config; struct spx_vector_iter *iter = spx_vector_iter_init(c->trackers,&(jc->err)); if(NULL == iter){ SpxLog2(jc->log,SpxLogError,jc->err,\ "init the trackers iter is fail."); return; } struct ydb_tracker *t = NULL; while(NULL != (t = spx_vector_iter_next(iter))){ ydb_storage_query_remote(c->log, t,c->groupname,c->machineid,c->syncgroup); } spx_vector_iter_free(&iter); ev_timer_set (sync_timer, (double) 30, 0.); ev_timer_again(sloop,sync_timer); }/*}}}*/
spx_private void *spx_thread_context_new(size_t idx,void *arg,err_t *err){/*{{{*/ SpxLogDelegate *log = (SpxLogDelegate *) arg; struct spx_thread_context *tc = (struct spx_thread_context *) \ spx_alloc_alone(sizeof(*tc),err); if(NULL == tc){ SpxLog2(log,SpxLogError,*err,\ "alloc thread context is fail."); return NULL; } tc->idx = idx; tc->loop = ev_loop_new(0); tc->log = log; if(-1 == pipe(tc->pipe)){ *err = errno; ev_loop_destroy(tc->loop); SpxFree(tc); return NULL; } spx_set_nb(tc->pipe[0]); spx_set_nb(tc->pipe[1]); return tc; }/*}}}*/
err_t StartLogDB(long (*ServerHandler)(size_t req_size, char *request, char **response), struct server_config *conf){ err_t err = 0; if(NULL == conf){ SpxLog1(g_log,SpxLogError, "start server failed. server config is NULL"); return -1; } if(0 != (err = NewCTXPool(conf, ServerHandler)) ){ SpxLog1(g_log, SpxLogError, "NewCTXPool failed."); return err; } pthread_t tid = NewServerThread(g_log, conf, &err); if(0 != err){ SpxLog2(g_log, SpxLogError, err, "create main socket thread is fail." "and will exit..."); return err; } pthread_join(tid,NULL); return 0; }
int main(int argc,char **argv) { err_t rc = 0; SpxLogDelegate *log = spx_log; struct spx_skiplist *spl = NULL; if(0 != (rc = spx_skiplist_new(spx_log,\ SPX_SKIPLIST_IDX_I32,15,false,\ spx_skiplist_i32_default_cmper,\ NULL,\ spx_skiplist_i32_default_printf,\ skiplist_kfree,\ skiplist_vfree,\ &spl))) { SpxLog2(log,SpxLogError,rc,\ "create skiplist is fail;"); return 0; } int i = 0; for(i = 1; i< 100; i++) { if(0 == i % 10) { i32_t *k = NULL; if(0 != (rc = spx_alloc_alone(sizeof(i32_t),(void **)&k))) { SpxLog2(log,SpxLogError,rc,\ "alloc for key is fail."); return 0; } *k = i; i32_t *v = NULL; if(0 != (rc = spx_alloc_alone(sizeof(i32_t),(void **)&v))) { SpxLog2(log,SpxLogError,rc,\ "alloc for value is fail."); return 0; } *v = *k; if(0 != (rc = spx_skiplist_insert(spl,k,sizeof(*k),v,sizeof(*v),-1))) { SpxLog2(log,SpxLogError,rc,\ "insert key to skiplist is fail."); } } } spx_skiplist_printf(spl); for(i = 0; i< 100; i++) { if(0 == i % 11) { i32_t *k = NULL; if(0 != (rc = spx_alloc_alone(sizeof(i32_t),(void **)&k))) { SpxLog2(log,SpxLogError,rc,\ "alloc for key is fail."); return 0; } *k = i; i32_t *v = NULL; if(0 != (rc = spx_alloc_alone(sizeof(i32_t),(void **)&v))) { SpxLog2(log,SpxLogError,rc,\ "alloc for value is fail."); return 0; } *v = *k; if(0 != (rc = spx_skiplist_insert(spl,k,sizeof(*k),v,sizeof(*v),-1))) { SpxLog2(log,SpxLogError,rc,\ "insert key to skiplist is fail."); } } } spx_skiplist_printf(spl); for(i = 0; i< 100; i++) { if(0 == i % 11) { if(0 != (rc = spx_skiplist_delete(spl,&i,sizeof(i)))) { SpxLog2(log,SpxLogError,rc,\ "delete the key:50 is fail."); } } } i32_t delk = 0; if(0 != (rc = spx_skiplist_delete(spl,&delk,sizeof(delk)))) { SpxLog2(log,SpxLogError,rc,\ "delete the key:50 is fail."); } spx_skiplist_printf(spl); /* delk = 10; if(0 != (rc = spx_skiplist_delete(spl,&delk,sizeof(delk)))){ SpxLog2(log,SpxLogError,rc,\ "delete the key:50 is fail."); } spx_skiplist_printf(spl); */ for(i = 0; i< 100; i++) { if(0 == i % 9) { i32_t *v = NULL; u64_t vl = 0; if(0 != (rc = spx_skiplist_get_and_move(spl,(void *)&i,(u32_t) sizeof(i),\ (void **) &v,&vl,\ skiplist_range_cmper))) { SpxLog2(log,SpxLogError,rc,\ "get and move key:9 is fail."); return 0; } } spx_skiplist_printf(spl); } return 0; }
void ydb_storage_dio_do_delete_form_chunkfile( struct ev_loop *loop,ev_async *w,int revents){/*{{{*/ ev_async_stop(loop,w); err_t err = 0; struct ydb_storage_dio_context *dc = (struct ydb_storage_dio_context *) w->data; struct spx_task_context *tc = dc->tc; struct spx_job_context *jc = dc->jc; struct ydb_storage_configurtion *c = jc->config; if(0 != (err = ydb_storage_dio_delete_context_from_chunkfile( c,dc->filename,dc->begin,dc->totalsize, dc->opver,dc->ver,dc->lastmodifytime, dc->realsize,spx_now()))){ SpxLogFmt2(dc->log,SpxLogError,err, "delete context begin:%lld,realsize:%lld,totalsize:%lld " "form chunkfile:%s is fail.", dc->begin,dc->realsize,dc->totalsize, dc->filename); goto r1; } YdbStorageBinlogDeleteWriter(dc->rfid); struct ydb_storage_mountpoint *mp = spx_list_get(c->mountpoints, dc->mp_idx); mp->last_modify_time = spx_now(); struct spx_msg_header *wh = (struct spx_msg_header *) \ spx_alloc_alone(sizeof(*wh),&err); if(NULL == wh){ SpxLogFmt2(dc->log,SpxLogError,err, "delete context begin:%lld,realsize:%lld,totalsize:%lld " "form chunkfile:%s is success bug new response header is fail.", dc->begin,dc->realsize,dc->totalsize, dc->filename); goto r1; } jc->writer_header = wh; wh->version = YDB_VERSION; wh->protocol = YDB_C2S_DELETE; wh->offset = 0; wh->bodylen = 0; err = 0; goto r2; r1: jc->writer_header = (struct spx_msg_header *) spx_alloc_alone(sizeof(*(jc->writer_header)),&err); if(NULL == jc->writer_header){ SpxLog2(dc->log,SpxLogError,err,\ "new response header is fail." "no notify client and push jc force."); spx_task_pool_push(g_spx_task_pool,tc); ydb_storage_dio_pool_push(g_ydb_storage_dio_pool,dc); spx_job_pool_push(g_spx_job_pool,jc); return; } jc->writer_header->protocol = jc->reader_header->protocol; jc->writer_header->bodylen = 0; jc->writer_header->version = YDB_VERSION; jc->writer_header->err = err; r2: spx_task_pool_push(g_spx_task_pool,tc); ydb_storage_dio_pool_push(g_ydb_storage_dio_pool,dc); jc->err = err; jc->moore = SpxNioMooreResponse; size_t idx = spx_network_module_wakeup_idx(jc); struct spx_thread_context *threadcontext = spx_get_thread(g_spx_network_module,idx); jc->tc = threadcontext; SpxModuleDispatch(spx_network_module_wakeup_handler,jc); return; }/*}}}*/
spx_private void ReciveRequest_GetRequest(int revents, void *arg){/*{{{*/ struct server_context *ctx = (struct server_context *) arg; if(NULL == ctx){ SpxLog1(g_log, SpxLogError, "ReciveRequest_GetRequest ctx is NULL"); return; } if(READ_HEADER == ctx->life_cycle){ if(EV_ERROR & revents){ SpxLog1(g_log, SpxLogError, "READ_HEADER EV_ERROR"); return; } if(EV_TIMEOUT & revents){ if((ctx->header_retry++) >= RETRY_TIMES){ RequestException(ctx, bad_request); SpxLog1(g_log, SpxLogError, "READ_HEADER EV_TIMEOUT"); return; } else{ ev_once(main_socket_loop, ctx->fd, EV_READ, ctx->timeout, ReciveRequest_GetRequest, ctx); return; } } if(EV_READ & revents){ char buf[LOG_HEADER_SIZE] = {0}; size_t len = 0; err_t err = ReciveRequest_GetRequest_ReadRequest(ctx->fd, buf, &len, LOG_HEADER_SIZE); if(0 == err){ log_header_unpack(buf, ctx->header); char * request = (char *) calloc(1, sizeof(char)*ctx->header->req_size); if(NULL == request){ SpxLog2(g_log, SpxLogError, err,"calloc log_header failed"); return; } ctx->req_size = ctx->header->req_size; ctx->request = request; ctx->life_cycle = READ_REQUEST; } else{ if((EAGAIN == err || EWOULDBLOCK == err || EINTR == err)) { ev_once(main_socket_loop, ctx->fd, EV_READ, ctx->timeout, ReciveRequest_GetRequest, ctx); return; } else{ SpxLog2(g_log, SpxLogError, err,"ReciveRequest_GetRequest_ReadRequest Failed"); CloseCTX(ctx); return; } } } } if(READ_REQUEST == ctx->life_cycle){ if(EV_ERROR & revents){ SpxLog1(g_log, SpxLogError, "READ_REQUEST EV_ERROR"); return; } if(EV_TIMEOUT & revents){ if((ctx->req_retry++) >= RETRY_TIMES){ RequestException(ctx, bad_request); SpxLog1(g_log, SpxLogError, "READ_REQUEST EV_TIMEOUT"); return; } else{ ev_once(main_socket_loop, ctx->fd, EV_READ, ctx->timeout, ReciveRequest_GetRequest, ctx); return; } } if(EV_READ & revents){ err_t err = ReciveRequest_GetRequest_ReadRequest(ctx->fd, ctx->request, &ctx->req_len, ctx->req_size); if(0 == err){ ctx->life_cycle = PARSE_REQUEST; RegisterAayncWatcher(&ctx->async_watcher, ParserRequest, ctx); ev_async_start(main_socket_loop, &ctx->async_watcher); ev_async_send(main_socket_loop, &ctx->async_watcher); } else{ if((EAGAIN == err || EWOULDBLOCK == err || EINTR == err)) { ev_once(main_socket_loop, ctx->fd, EV_READ, ctx->timeout, ReciveRequest_GetRequest, ctx); return; } else{ SpxLog2(g_log, SpxLogError, err,"ReciveRequest_GetRequest_ReadRequest Failed"); CloseCTX(ctx); return; } } } } }/*}}}*/
err_t ydb_storage_dio_delete_context_from_chunkfile( struct ydb_storage_configurtion *c, string_t fname, u64_t cbegin, u64_t ctotalsize, u32_t copver, u32_t cver, u64_t clastmodifytime, u64_t crealsize, u64_t lastmodifytime){/*{{{*/ err_t err = 0; struct spx_msg *ioctx = NULL; int fd = 0; char *mptr = NULL; string_t io_suffix = NULL; string_t io_hashcode = NULL; bool_t io_isdelete = false; u32_t io_opver = 0; u32_t io_ver = 0; u64_t io_createtime = 0; u64_t io_lastmodifytime = 0; u64_t io_totalsize = 0; u64_t io_realsize = 0; u32_t unit = (int) cbegin / c->pagesize; u64_t begin = unit * c->pagesize; u64_t offset = cbegin - begin; u64_t len = offset + ctotalsize; fd = open(fname,O_RDWR,SpxFileMode); if(0 == fd){ err = errno; SpxLogFmt2(c->log,SpxLogError,err,\ "open chunkfile:%s is fail.", fname); goto r1; } mptr = mmap(NULL,\ len,PROT_READ | PROT_WRITE ,\ MAP_SHARED,fd,begin); if(MAP_FAILED == mptr){ err = errno; SpxLogFmt2(c->log,SpxLogError,err,\ "mmap chunkfile:%s with begin:%lld,length:%lld is fail.", fname,begin,len); goto r1; } ioctx = spx_msg_new(YDB_CHUNKFILE_MEMADATA_SIZE,&err); if(NULL == ioctx){ SpxLog2(c->log,SpxLogError,err,\ "alloc metedata ioctx is fail."); goto r1; } if(0 != (err = spx_msg_pack_ubytes(ioctx, ((ubyte_t *) (mptr+ offset)), YDB_CHUNKFILE_MEMADATA_SIZE))){ SpxLog2(c->log,SpxLogError,err,\ "read metedata to ioctx is fail."); goto r1; } spx_msg_seek(ioctx,0,SpxMsgSeekSet); err = ydb_storage_dio_parser_metadata(c->log,ioctx, &io_isdelete,&io_opver, &io_ver,&io_createtime, &io_lastmodifytime,&io_totalsize,&io_realsize, &io_suffix,&io_hashcode); if(0 != err){ SpxLog2(c->log,SpxLogError,err,\ "unpack metedate ioctx is fail."); goto r1; } if(io_isdelete){ SpxLogFmt1(c->log,SpxLogError, "the file in the chunkfile:%s " "with begin:%lld totalsize:%lld is deleted.", fname,cbegin,ctotalsize); err = ENOENT; goto r1; } if(copver != io_opver || cver != io_ver || clastmodifytime != io_lastmodifytime || ctotalsize != io_totalsize || crealsize != io_realsize){ SpxLogFmt1(c->log,SpxLogError, "the file in the chunkfile:%s " "with begin:%lld totalsize:%lld is not the deleting-file.", fname,cbegin,ctotalsize); err = ENOENT; goto r1; } spx_msg_seek(ioctx,0,SpxMsgSeekSet); spx_msg_pack_true(ioctx);//isdelete spx_msg_pack_u32(ioctx,copver + 1); spx_msg_pack_u32(ioctx,YDB_VERSION); spx_msg_seek(ioctx,sizeof(u64_t),SpxMsgSeekCurrent);//jump createtime spx_msg_pack_u64(ioctx,lastmodifytime); memcpy(mptr + offset,ioctx->buf,YDB_CHUNKFILE_MEMADATA_SIZE); r1: if(NULL != ioctx){ SpxMsgFree(ioctx); } if(NULL != mptr){ munmap(mptr,len); } if(0 < fd){ SpxClose(fd); } if(NULL != io_hashcode){ SpxStringFree(io_hashcode); } if(NULL != io_suffix){ SpxStringFree(io_suffix); } return err; }/*}}}*/
err_t ydb_storage_dio_delete(struct ev_loop *loop,\ struct ydb_storage_dio_context *dc){/*{{{*/ err_t err = 0; struct spx_task_context *tc = dc->tc; struct spx_job_context *jc = dc->jc; struct ydb_storage_configurtion *c = jc->config; struct spx_msg *ctx = jc->reader_body_ctx; size_t len = jc->reader_header->bodylen; dc->rfid = spx_msg_unpack_string(ctx,len,&(err)); if(NULL == dc->rfid){ SpxLogFmt2(c->log,SpxLogError,err,\ "unpack the fid from client:%s is fail.", jc->client_ip); goto r1; } if(0 != ( err = ydb_storage_dio_parser_fileid(jc->log,dc->rfid, &(dc->groupname),&(dc->machineid),&(dc->syncgroup), &(dc->issinglefile),&(dc->mp_idx),&(dc->p1), &(dc->p2),&(dc->tidx),&(dc->file_createtime), &(dc->rand),&(dc->begin),&(dc->realsize), &(dc->totalsize),&(dc->ver),&(dc->opver), &(dc->lastmodifytime),&(dc->hashcode), &(dc->has_suffix),&(dc->suffix)))){ SpxLogFmt2(dc->log,SpxLogError,err,\ "parser fid:%s from client:%s is fail.", dc->rfid,jc->client_ip); goto r1; } dc->filename = ydb_storage_dio_make_filename(dc->log,dc->issinglefile, c->mountpoints, dc->mp_idx, dc->p1,dc->p2, dc->machineid,dc->tidx,dc->file_createtime, dc->rand,dc->suffix,&err); if(NULL == dc->filename){ SpxLogFmt2(dc->log,SpxLogError,err,\ "make filename with fid:%s from client:%s is fail.", dc->rfid,jc->client_ip); goto r1; } if(!SpxFileExist(dc->filename)) { SpxLogFmt1(dc->log,SpxLogWarn,\ "deleting-file:%s is not exist.request from:%s.", dc->filename,jc->client_ip); goto r1; } if(dc->issinglefile){ if(0 != remove(dc->filename)){ err = errno; SpxLogFmt2(dc->log,SpxLogError,err,\ "delete file :%s is fail.request from:%s.", dc->filename,jc->client_ip); } YdbStorageBinlogDeleteWriter(dc->rfid); struct ydb_storage_mountpoint *mp = spx_list_get(c->mountpoints, dc->mp_idx); mp->last_modify_time = spx_now(); goto r1; } else { spx_dio_regedit_async(&(dc->async), ydb_storage_dio_do_delete_form_chunkfile,dc); ev_async_start(loop,&(dc->async)); ev_async_send(loop,&(dc->async)); } return err; r1: spx_task_pool_push(g_spx_task_pool,tc); ydb_storage_dio_pool_push(g_ydb_storage_dio_pool,dc); jc->writer_header = (struct spx_msg_header *) spx_alloc_alone(sizeof(*(jc->writer_header)),&err); if(NULL == jc->writer_header){ SpxLog2(dc->log,SpxLogError,err,\ "new response header is fail." "no notify client and push jc force."); spx_job_pool_push(g_spx_job_pool,jc); return err; } jc->writer_header->protocol = jc->reader_header->protocol; jc->writer_header->bodylen = 0; jc->writer_header->version = YDB_VERSION; jc->writer_header->err = err; jc->err = err; jc->moore = SpxNioMooreResponse; size_t idx = spx_network_module_wakeup_idx(jc); struct spx_thread_context *threadcontext = spx_get_thread(g_spx_network_module,idx); jc->tc = threadcontext; SpxModuleDispatch(spx_network_module_wakeup_handler,jc); return 0; }/*}}}*/
pthread_t ydb_storage_syncquery_init( SpxLogDelegate *log, u32_t timeout, struct ydb_storage_configurtion *config,\ err_t *err){/*{{{*/ struct spx_job_context_transport arg; SpxZero(arg); arg.timeout = timeout; arg.nio_reader = spx_nio_reader; arg.nio_writer = spx_nio_writer; arg.log = log; arg.reader_body_process = ydb_storage_syncquery_nio_body_reader; arg.writer_body_process = ydb_storage_syncquery_nio_body_writer; arg.config = config; sync_timer = spx_alloc_alone(sizeof(*sync_timer),err); if(NULL == sync_timer){ SpxLog2(log,SpxLogError,*err,"alloc sync timer is fail."); goto r1; } sloop = ev_loop_new(0); if(NULL == sloop){ *err = errno; SpxLog2(log,SpxLogError,*err,\ "new event loop for sync is fail."); goto r1; } if(!ydb_storage_get_remote(config,&arg)){ goto r1; } pthread_t tid = 0; pthread_attr_t attr; pthread_attr_init(&attr); size_t ostack_size = 0; pthread_attr_getstacksize(&attr, &ostack_size); do{ if (ostack_size != config->stacksize && (0 != (*err = pthread_attr_setstacksize(&attr,config->stacksize)))){ pthread_attr_destroy(&attr); SpxLog2(log,SpxLogError,*err,\ "set thread stack size is fail."); goto r1; } if (0 !=(*err = pthread_create(&(tid), &attr, ydb_storage_syncquery_heartbeat, NULL))){ pthread_attr_destroy(&attr); SpxLog2(log,SpxLogError,*err,\ "create heartbeat thread is fail."); goto r1; } }while(false); pthread_attr_destroy(&attr); return tid; r1: if(NULL != sync_timer){ SpxFree(sync_timer); } return 0; }/*}}}*/
struct spx_module_context *spx_module_new(\ SpxLogDelegate *log,\ u32_t threadsize,\ size_t stack_size, SpxReceiveTriggerDelegate *receive_handler, err_t *err){/*{{{*/ struct spx_module_context *mc = (struct spx_module_context *)\ spx_alloc_alone(sizeof(*mc),err); if(NULL == mc){ SpxLog2(log,SpxLogError,*err,\ "alloc module context is fail."); return NULL; } mc->log = log; mc->threadpool = spx_list_init(log,\ threadsize,\ spx_thread_context_new,\ log,\ spx_thread_context_free,\ err); if(NULL == mc->threadpool){ SpxLog2(log,SpxLogError,*err,\ "alloc threadpool for module is fail."); goto r2; } struct spx_recvive_context_transport rct; SpxZero(rct); rct.log = log; rct.event = EV_READ; rct.recviver = receive_handler;; mc->receive_triggers = spx_list_init(log,\ threadsize,\ spx_receiver_new,\ &rct,\ spx_receiver_free,\ err); if(NULL == mc->receive_triggers){ SpxLog2(log,SpxLogError,*err,\ "alloc receive triggers are fail."); goto r2; } pthread_attr_t attr; pthread_attr_init(&attr); size_t ostack_size = 0; pthread_attr_getstacksize(&attr, &ostack_size); do{ if (ostack_size != stack_size && (0 != (*err = pthread_attr_setstacksize(&attr,stack_size)))){ SpxLog2(log,SpxLogError,*err,\ "set thread stack size is fail."); goto r1; } u32_t i = 0; struct spx_thread_pending_transport *tpt; for( ; i < threadsize; i++){ tpt = spx_alloc_alone(sizeof(*tpt),err); tpt->mc = mc; tpt->idx = i; struct spx_thread_context *n = spx_list_get(mc->threadpool,i); if (0 !=(*err = pthread_create(&(n->tid), &attr, spx_thread_listening, tpt))){ SpxLog2(log,SpxLogError,*err,\ "create nio thread is fail."); goto r1; } } }while(false); pthread_attr_destroy(&attr); return mc; r1: pthread_attr_destroy(&attr); r2: spx_module_free(&mc); return NULL; }/*}}}*/