int start_zookeeper(view* cur_view, int *zfd, pthread_mutex_t *lock)
{
int rc;
char zoo_host_port[32];
sprintf(zoo_host_port, "localhost:%d", zoo_port);
zh = zookeeper_init(zoo_host_port, zookeeper_init_watcher, 15000, 0, 0, 0);
while(is_connected != 1);
int interest, fd;
struct timeval tv;
zookeeper_interest(zh, &fd, &interest, &tv);
*zfd = fd;
char path_buffer[512];
rc = zoo_create(zh, "/election/guid-n_", NULL, -1, &ZOO_OPEN_ACL_UNSAFE, ZOO_SEQUENCE|ZOO_EPHEMERAL, path_buffer, 512);
if (rc)
{
fprintf(stderr, "Error %d for zoo_create\n", rc);
}
char znode_path[512];
get_znode_path(path_buffer, znode_path);
check_leader(cur_view, path_buffer);
struct watcherContext *watcherPara = (struct watcherContext *)malloc(sizeof(struct watcherContext));
strcpy(watcherPara->znode_path, znode_path);
watcherPara->lock = lock;
watcherPara->cur_view = cur_view;
rc = zoo_wget_children(zh, "/election", zoo_wget_children_watcher, (void*)watcherPara, NULL);
if (rc)
{
fprintf(stderr, "Error %d for zoo_wget_children\n", rc);
}
return 0;
}
void ZookeeperClient::CheckConn()
{
int interest;
struct timeval tv;
int fd;
int rc = zookeeper_interest(m_zk, &fd, &interest, &tv);
if (rc == ZOK)
{
if (fd != -1)
{
int events = 0;
if (interest & ZOOKEEPER_READ)
{
events |= AE_READABLE;
}
if (interest & ZOOKEEPER_WRITE)
{
events |= AE_WRITABLE;
}
aeDeleteFileEvent(m_serv.GetRawEventLoop(), fd,
AE_READABLE | AE_WRITABLE);
aeCreateFileEvent(m_serv.GetRawEventLoop(), fd, events,
ZookeeperClient::ZKIOCallback, this);
m_zk_fd = fd;
}
int64 wait = tv.tv_sec * 1000 + tv.tv_usec / 1000;
if (wait == 0)
{
wait = 1000;
}
if (m_timer_task_id == -1)
{
m_timer_task_id = m_serv.GetTimer().Schedule(this, wait, -1,
MILLIS);
}
}
else
{
if (rc != ZOK)
{
if (m_timer_task_id == -1)
{
m_timer_task_id = m_serv.GetTimer().Schedule(this, 1000, -1,
MILLIS);
}
}
}
}
/**
* register zk with libevent for callbacks. This gets driven through the
* default timeout from zookeeper_interest (and subsequently whether there is
* need to read/write data.
*/
static void trigger_event()
{
int interest = 0;
struct timeval tv;
if (zh) {
int res = zookeeper_interest(zh, &fd, &interest, &tv);
if (res < 0) {
LOG_WARN(("Error while checking zookeeper: %d", res));
}
else {
// Always run with timeout.
short event_flags = EV_TIMEOUT;
if (fd != -1) {
if (interest & ZOOKEEPER_READ) {
LOG_DEBUG(("READ ->"));
event_flags |= EV_READ;
}
if (interest & ZOOKEEPER_WRITE) {
LOG_DEBUG(("WRITE ->"));
event_flags |= EV_WRITE;
}
}
else {
LOG_DEBUG(("Not interested in anything right now..."));
}
LOG_DEBUG(("Triggering event (%d/%d)", tv.tv_sec, tv.tv_usec));
event_set(&zk_event, fd, event_flags, zookeeper_event_handler, NULL);
event_add(&zk_event, &tv);
}
}
else {
LOG_DEBUG(("No zookeeper handle!"));
}
}
int adaptor_init(zhandle_t *zh)
{
pthread_mutexattr_t recursive_mx_attr;
struct adaptor_threads *adaptor_threads = calloc(1, sizeof(*adaptor_threads));
if (!adaptor_threads) {
LOG_ERROR(("Out of memory"));
return -1;
}
/* We use a pipe for interrupting select() in unix/sol and socketpair in windows. */
#ifdef WIN32
if (create_socket_pair(adaptor_threads->self_pipe) == -1){
LOG_ERROR(("Can't make a socket."));
#else
if(pipe(adaptor_threads->self_pipe)==-1) {
LOG_ERROR(("Can't make a pipe %d",errno));
#endif
free(adaptor_threads);
return -1;
}
set_nonblock(adaptor_threads->self_pipe[1]);
set_nonblock(adaptor_threads->self_pipe[0]);
pthread_mutex_init(&zh->auth_h.lock,0);
zh->adaptor_priv = adaptor_threads;
pthread_mutex_init(&zh->to_process.lock,0);
pthread_mutex_init(&adaptor_threads->zh_lock,0);
pthread_mutex_init(&adaptor_threads->reconfig_lock,0);
// to_send must be recursive mutex
pthread_mutexattr_init(&recursive_mx_attr);
pthread_mutexattr_settype(&recursive_mx_attr, PTHREAD_MUTEX_RECURSIVE);
pthread_mutex_init(&zh->to_send.lock,&recursive_mx_attr);
pthread_mutexattr_destroy(&recursive_mx_attr);
pthread_mutex_init(&zh->sent_requests.lock,0);
pthread_cond_init(&zh->sent_requests.cond,0);
pthread_mutex_init(&zh->completions_to_process.lock,0);
pthread_cond_init(&zh->completions_to_process.cond,0);
start_threads(zh);
return 0;
}
void adaptor_finish(zhandle_t *zh)
{
struct adaptor_threads *adaptor_threads;
// make sure zh doesn't get destroyed until after we're done here
api_prolog(zh);
adaptor_threads = zh->adaptor_priv;
if(adaptor_threads==0) {
api_epilog(zh,0);
return;
}
if(!pthread_equal(adaptor_threads->io,pthread_self())){
wakeup_io_thread(zh);
pthread_join(adaptor_threads->io, 0);
}else
pthread_detach(adaptor_threads->io);
if(!pthread_equal(adaptor_threads->completion,pthread_self())){
pthread_mutex_lock(&zh->completions_to_process.lock);
pthread_cond_broadcast(&zh->completions_to_process.cond);
pthread_mutex_unlock(&zh->completions_to_process.lock);
pthread_join(adaptor_threads->completion, 0);
}else
pthread_detach(adaptor_threads->completion);
api_epilog(zh,0);
}
void adaptor_destroy(zhandle_t *zh)
{
struct adaptor_threads *adaptor = zh->adaptor_priv;
if(adaptor==0) return;
pthread_cond_destroy(&adaptor->cond);
pthread_mutex_destroy(&adaptor->lock);
pthread_mutex_destroy(&zh->to_process.lock);
pthread_mutex_destroy(&zh->to_send.lock);
pthread_mutex_destroy(&zh->sent_requests.lock);
pthread_cond_destroy(&zh->sent_requests.cond);
pthread_mutex_destroy(&zh->completions_to_process.lock);
pthread_cond_destroy(&zh->completions_to_process.cond);
pthread_mutex_destroy(&adaptor->zh_lock);
pthread_mutex_destroy(&zh->auth_h.lock);
close(adaptor->self_pipe[0]);
close(adaptor->self_pipe[1]);
free(adaptor);
zh->adaptor_priv=0;
}
int wakeup_io_thread(zhandle_t *zh)
{
struct adaptor_threads *adaptor_threads = zh->adaptor_priv;
char c=0;
#ifndef WIN32
return write(adaptor_threads->self_pipe[1],&c,1)==1? ZOK: ZSYSTEMERROR;
#else
return send(adaptor_threads->self_pipe[1], &c, 1, 0)==1? ZOK: ZSYSTEMERROR;
#endif
}
int adaptor_send_queue(zhandle_t *zh, int timeout)
{
if(!zh->close_requested)
return wakeup_io_thread(zh);
// don't rely on the IO thread to send the messages if the app has
// requested to close
return flush_send_queue(zh, timeout);
}
/* These two are declared here because we will run the event loop
* and not the client */
#ifdef WIN32
int zookeeper_interest(zhandle_t *zh, SOCKET *fd, int *interest,
struct timeval *tv);
#else
int zookeeper_interest(zhandle_t *zh, int *fd, int *interest,
struct timeval *tv);
#endif
int zookeeper_process(zhandle_t *zh, int events);
#ifdef WIN32
unsigned __stdcall do_io( void * v)
#else
void *do_io(void *v)
#endif
{
zhandle_t *zh = (zhandle_t*)v;
#ifndef WIN32
struct pollfd fds[2];
struct adaptor_threads *adaptor_threads = zh->adaptor_priv;
api_prolog(zh);
notify_thread_ready(zh);
LOG_DEBUG(("started IO thread"));
fds[0].fd=adaptor_threads->self_pipe[0];
fds[0].events=POLLIN;
while(!zh->close_requested) {
struct timeval tv;
int fd;
int interest;
int timeout;
int maxfd=1;
int rc;
zookeeper_interest(zh, &fd, &interest, &tv);
if (fd != -1) {
fds[1].fd=fd;
fds[1].events=(interest&ZOOKEEPER_READ)?POLLIN:0;
fds[1].events|=(interest&ZOOKEEPER_WRITE)?POLLOUT:0;
maxfd=2;
}
timeout=tv.tv_sec * 1000 + (tv.tv_usec/1000);
poll(fds,maxfd,timeout);
if (fd != -1) {
interest=(fds[1].revents&POLLIN)?ZOOKEEPER_READ:0;
interest|=((fds[1].revents&POLLOUT)||(fds[1].revents&POLLHUP))?ZOOKEEPER_WRITE:0;
}
if(fds[0].revents&POLLIN){
// flush the pipe
char b[128];
while(read(adaptor_threads->self_pipe[0],b,sizeof(b))==sizeof(b)){}
}
#else
fd_set rfds, wfds, efds;
struct adaptor_threads *adaptor_threads = zh->adaptor_priv;
api_prolog(zh);
notify_thread_ready(zh);
LOG_DEBUG(("started IO thread"));
FD_ZERO(&rfds); FD_ZERO(&wfds); FD_ZERO(&efds);
while(!zh->close_requested) {
struct timeval tv;
SOCKET fd;
SOCKET maxfd=adaptor_threads->self_pipe[0];
int interest;
int rc;
zookeeper_interest(zh, &fd, &interest, &tv);
if (fd != -1) {
if (interest&ZOOKEEPER_READ) {
FD_SET(fd, &rfds);
} else {
FD_CLR(fd, &rfds);
}
if (interest&ZOOKEEPER_WRITE) {
FD_SET(fd, &wfds);
} else {
FD_CLR(fd, &wfds);
}
}
FD_SET( adaptor_threads->self_pipe[0] ,&rfds );
rc = select((int)maxfd, &rfds, &wfds, &efds, &tv);
if (fd != -1)
{
interest = (FD_ISSET(fd, &rfds))? ZOOKEEPER_READ:0;
interest|= (FD_ISSET(fd, &wfds))? ZOOKEEPER_WRITE:0;
}
if (FD_ISSET(adaptor_threads->self_pipe[0], &rfds)){
// flush the pipe/socket
char b[128];
while(recv(adaptor_threads->self_pipe[0],b,sizeof(b), 0)==sizeof(b)){}
}
#endif
// dispatch zookeeper events
rc = zookeeper_process(zh, interest);
// check the current state of the zhandle and terminate
// if it is_unrecoverable()
if(is_unrecoverable(zh))
break;
}
api_epilog(zh, 0);
LOG_DEBUG(("IO thread terminated"));
return 0;
}
#ifdef WIN32
unsigned __stdcall do_completion( void * v)
#else
void *do_completion(void *v)
#endif
{
zhandle_t *zh = v;
api_prolog(zh);
notify_thread_ready(zh);
LOG_DEBUG(("started completion thread"));
while(!zh->close_requested) {
pthread_mutex_lock(&zh->completions_to_process.lock);
while(!zh->completions_to_process.head && !zh->close_requested) {
pthread_cond_wait(&zh->completions_to_process.cond, &zh->completions_to_process.lock);
}
pthread_mutex_unlock(&zh->completions_to_process.lock);
process_completions(zh);
}
api_epilog(zh, 0);
LOG_DEBUG(("completion thread terminated"));
return 0;
}
int main(int argc, char **argv) {
#ifndef THREADED
fd_set rfds, wfds, efds;
int processed=0;
#endif
char buffer[4096];
char p[2048];
#ifdef YCA
char *cert=0;
char appId[64];
#endif
int bufoff = 0;
FILE *fh;
if (argc < 2) {
fprintf(stderr,
"USAGE %s zookeeper_host_list [clientid_file|cmd:(ls|ls2|create|od|...)]\n",
argv[0]);
fprintf(stderr,
"Version: ZooKeeper cli (c client) version %d.%d.%d\n",
ZOO_MAJOR_VERSION,
ZOO_MINOR_VERSION,
ZOO_PATCH_VERSION);
return 2;
}
if (argc > 2) {
if(strncmp("cmd:",argv[2],4)==0){
size_t cmdlen = strlen(argv[2]);
if (cmdlen > sizeof(cmd)) {
fprintf(stderr,
"Command length %zu exceeds max length of %zu\n",
cmdlen,
sizeof(cmd));
return 2;
}
strncpy(cmd, argv[2]+4, sizeof(cmd));
batchMode=1;
fprintf(stderr,"Batch mode: %s\n",cmd);
}else{
clientIdFile = argv[2];
fh = fopen(clientIdFile, "r");
if (fh) {
if (fread(&myid, sizeof(myid), 1, fh) != sizeof(myid)) {
memset(&myid, 0, sizeof(myid));
}
fclose(fh);
}
}
}
#ifdef YCA
strcpy(appId,"yahoo.example.yca_test");
cert = yca_get_cert_once(appId);
if(cert!=0) {
fprintf(stderr,"Certificate for appid [%s] is [%s]\n",appId,cert);
strncpy(p,cert,sizeof(p)-1);
free(cert);
} else {
fprintf(stderr,"Certificate for appid [%s] not found\n",appId);
strcpy(p,"dummy");
}
#else
strcpy(p, "dummy");
#endif
verbose = 0;
zoo_set_debug_level(ZOO_LOG_LEVEL_WARN);
zoo_deterministic_conn_order(1); // enable deterministic order
hostPort = argv[1];
zh = zookeeper_init(hostPort, watcher, 30000, &myid, 0, 0);
if (!zh) {
return errno;
}
#ifdef YCA
if(zoo_add_auth(zh,"yca",p,strlen(p),0,0)!=ZOK)
return 2;
#endif
#ifdef THREADED
while(!shutdownThisThing) {
int rc;
int len = sizeof(buffer) - bufoff -1;
if (len <= 0) {
fprintf(stderr, "Can't handle lines that long!\n");
exit(2);
}
rc = read(0, buffer+bufoff, len);
if (rc <= 0) {
fprintf(stderr, "bye\n");
shutdownThisThing=1;
break;
}
bufoff += rc;
buffer[bufoff] = '\0';
while (strchr(buffer, '\n')) {
char *ptr = strchr(buffer, '\n');
*ptr = '\0';
processline(buffer);
ptr++;
memmove(buffer, ptr, strlen(ptr)+1);
bufoff = 0;
}
}
#else
FD_ZERO(&rfds);
FD_ZERO(&wfds);
FD_ZERO(&efds);
while (!shutdownThisThing) {
int fd;
int interest;
int events;
struct timeval tv;
int rc;
zookeeper_interest(zh, &fd, &interest, &tv);
if (fd != -1) {
if (interest&ZOOKEEPER_READ) {
FD_SET(fd, &rfds);
} else {
FD_CLR(fd, &rfds);
}
if (interest&ZOOKEEPER_WRITE) {
FD_SET(fd, &wfds);
} else {
FD_CLR(fd, &wfds);
}
} else {
fd = 0;
}
FD_SET(0, &rfds);
rc = select(fd+1, &rfds, &wfds, &efds, &tv);
events = 0;
if (rc > 0) {
if (FD_ISSET(fd, &rfds)) {
events |= ZOOKEEPER_READ;
}
if (FD_ISSET(fd, &wfds)) {
events |= ZOOKEEPER_WRITE;
}
}
if(batchMode && processed==0){
//batch mode
processline(cmd);
processed=1;
}
if (FD_ISSET(0, &rfds)) {
int rc;
int len = sizeof(buffer) - bufoff -1;
if (len <= 0) {
fprintf(stderr, "Can't handle lines that long!\n");
exit(2);
}
rc = read(0, buffer+bufoff, len);
if (rc <= 0) {
fprintf(stderr, "bye\n");
break;
}
bufoff += rc;
buffer[bufoff] = '\0';
while (strchr(buffer, '\n')) {
char *ptr = strchr(buffer, '\n');
*ptr = '\0';
processline(buffer);
ptr++;
memmove(buffer, ptr, strlen(ptr)+1);
bufoff = 0;
}
}
zookeeper_process(zh, events);
}
#endif
if (to_send!=0)
fprintf(stderr,"Recvd %d responses for %d requests sent\n",recvd,sent);
zookeeper_close(zh);
return 0;
}
int main(int argc, char *argv[])
{
int rc;
int fd;
int interest;
int events;
struct timeval tv;
fd_set rfds, wfds, efds;
if (argc != 2) {
fprintf(stderr, "USAGE: %s host:port\n", argv[0]);
exit(1);
}
FD_ZERO(&rfds);
FD_ZERO(&wfds);
FD_ZERO(&efds);
zoo_set_debug_level(ZOO_LOG_LEVEL_INFO);
zoo_deterministic_conn_order(1);
hostPort = argv[1];
zh = zookeeper_init(hostPort, watcher, 30000, &myid, 0, 0);
if (!zh) {
return errno;
}
while (1) {
zookeeper_interest(zh, &fd, &interest, &tv);
usleep(10);
if (connected == 1) {
struct String_vector str;
usleep(10);
// watch existence of the node
rc = zoo_wget_children(zh, "/testpath1",
watchchildren , mycontext, &str);
if (ZOK != rc){
printf("Problems %d\n", rc);
} else {
int i = 0;
while (i < str.count) {
printf("Children %s\n", str.data[i++]);
}
if (str.count) {
deallocate_String_vector(&str);
}
}
connected++;
}
if (fd != -1) {
if (interest & ZOOKEEPER_READ) {
FD_SET(fd, &rfds);
} else {
FD_CLR(fd, &rfds);
}
if (interest & ZOOKEEPER_WRITE) {
FD_SET(fd, &wfds);
} else {
FD_CLR(fd, &wfds);
}
} else {
fd = 0;
}
FD_SET(0, &rfds);
rc = select(fd+1, &rfds, &wfds, &efds, &tv);
events = 0;
if (rc > 0) {
if (FD_ISSET(fd, &rfds)) {
events |= ZOOKEEPER_READ;
}
if (FD_ISSET(fd, &wfds)) {
events |= ZOOKEEPER_WRITE;
}
}
zookeeper_process(zh, events);
}
return 0;
}
int main (int argc, char * argv[]) {
LOG_DEBUG(("THREADED defined"));
if (argc != 2) {
fprintf(stderr, "USAGE: %s host:port\n", argv[0]);
exit(1);
}
/*
* Initialize ZooKeeper session
*/
if(init(argv[1])){
LOG_ERROR(("Error while initializing the master: ", errno));
}
#ifdef THREADED
/*
* Wait until connected
*/
while(!is_connected()) {
sleep(1);
}
LOG_DEBUG(("Connected, going to bootstrap and run for master"));
/*
* Create parent znodes
*/
bootstrap();
/*
* Run for master
*/
run_for_master();
/*
* Run until session expires
*/
while(!is_expired()) {
sleep(1);
}
#else
int run = 0;
fd_set rfds, wfds, efds;
FD_ZERO(&rfds);
FD_ZERO(&wfds);
FD_ZERO(&efds);
while (!is_expired()) {
int fd = -1;
int interest = 0;
int events ;
struct timeval tv;
int rc;
zookeeper_interest(zh, &fd, &interest, &tv);
if (fd != -1) {
if (interest&ZOOKEEPER_READ) {
FD_SET(fd, &rfds);
} else {
FD_CLR(fd, &rfds);
}
if (interest&ZOOKEEPER_WRITE) {
FD_SET(fd, &wfds);
} else {
FD_CLR(fd, &wfds);
}
} else {
fd = 0;
}
/*
* The next if block contains
* calls to bootstrap the master
* and run for master. We only
* get into it when the client
* has established a session and
* is_connected is true.
*/
if(is_connected() && !run) {
LOG_DEBUG(("Connected, going to bootstrap and run for master"));
/*
* Create parent znodes
*/
bootstrap();
/*
* Run for master
*/
run_for_master();
run = 1;
}
rc = select(fd+1, &rfds, &wfds, &efds, &tv);
events = 0;
if (rc > 0) {
if (FD_ISSET(fd, &rfds)) {
events |= ZOOKEEPER_READ;
}
if (FD_ISSET(fd, &wfds)) {
events |= ZOOKEEPER_WRITE;
}
}
zookeeper_process(zh, events);
}
#endif
return 0;
}
static VALUE method_zkrb_iterate_event_loop(VALUE self) {
FETCH_DATA_PTR(self, zk);
fd_set rfds, wfds, efds;
FD_ZERO(&rfds); FD_ZERO(&wfds); FD_ZERO(&efds);
int fd=0, interest=0, events=0, rc=0, maxfd=0;
struct timeval tv;
zookeeper_interest(zk->zh, &fd, &interest, &tv);
if (fd != -1) {
if (interest & ZOOKEEPER_READ) {
FD_SET(fd, &rfds);
} else {
FD_CLR(fd, &rfds);
}
if (interest & ZOOKEEPER_WRITE) {
FD_SET(fd, &wfds);
} else {
FD_CLR(fd, &wfds);
}
} else {
fd = 0;
}
// add our self-pipe to the read set, allow us to wake up in case our attention is needed
int pipe_r_fd = get_self_pipe_read_fd(self);
FD_SET(pipe_r_fd, &rfds);
maxfd = (pipe_r_fd > fd) ? pipe_r_fd : fd;
rc = rb_thread_select(maxfd+1, &rfds, &wfds, &efds, &tv);
if (rc > 0) {
if (FD_ISSET(fd, &rfds)) {
events |= ZOOKEEPER_READ;
}
if (FD_ISSET(fd, &wfds)) {
events |= ZOOKEEPER_WRITE;
}
// we got woken up by the self-pipe
if (FD_ISSET(pipe_r_fd, &rfds)) {
// one event has awoken us, so we clear one event from the pipe
char b[1];
if (read(pipe_r_fd, b, 1) < 0) {
rb_raise(rb_eRuntimeError, "read from pipe failed: %s", clean_errno());
}
}
rc = zookeeper_process(zk->zh, events);
}
else if (rc == 0) {
zkrb_debug("timed out waiting for descriptor to be ready");
}
else {
log_err("select returned: %d", rc);
}
return INT2FIX(rc);
}
