void grok_input_eof_handler(int fd, short what, void *data) {
grok_input_t *ginput = (grok_input_t *)data;
grok_program_t *gprog = ginput->gprog;
if (ginput->instance_match_count == 0) {
/* execute nomatch if there is one on this program */
grok_matchconfig_exec_nomatch(gprog, ginput);
}
switch (ginput->type) {
case I_PROCESS:
if (PROCESS_SHOULD_RESTART(&(ginput->source.process))) {
ginput->instance_match_count = 0;
event_once(-1, EV_TIMEOUT, _program_process_start, ginput,
&ginput->restart_delay);
} else {
grok_log(ginput->gprog, LOG_PROGRAM, "Not restarting process: %s",
ginput->source.process.cmd);
bufferevent_disable(ginput->bev, EV_READ);
close(ginput->source.process.p_stdin);
close(ginput->source.process.p_stdout);
close(ginput->source.process.p_stderr);
ginput->done = 1;
}
break;
case I_FILE:
if (ginput->source.file.follow) {
ginput->instance_match_count = 0;
event_once(-1, EV_TIMEOUT, _program_file_repair_event, ginput,
&ginput->restart_delay);
} else {
grok_log(ginput->gprog, LOG_PROGRAM, "Not restarting file: %s",
ginput->source.file.filename);
bufferevent_disable(ginput->bev, EV_READ);
close(ginput->source.file.reader);
close(ginput->source.file.writer);
close(ginput->source.file.fd);
ginput->done = 1;
}
break;
}
/* If all inputs are now done, close the shell */
int still_open = 0;
int i = 0;
for (i = 0; i < gprog->ninputs; i++) {
still_open += !gprog->inputs[i].done;
if (!gprog->inputs[i].done) {
grok_log(gprog, LOG_PROGRAM, "Input still open: %d", i);
}
}
if (still_open == 0) {
for (i = 0; i < gprog->nmatchconfigs; i++) {
grok_matchconfig_close(gprog, &gprog->matchconfigs[i]);
}
grok_collection_check_end_state(gprog->gcol);
}
}
void
privmsg_done(struct dht_rpc * rpc,
struct dht_group_msg_reply *reply,
void * arg)
{
struct dht_group *group = arg;
char *channel_name;
unsigned int error_code;
char *error_reason;
struct timeval tv;
if (reply == NULL)
return;
CU_ASSERT(!EVTAG_GET(reply, channel_name, &channel_name));
CU_ASSERT(!EVTAG_GET(reply, error_code, &error_code));
CU_ASSERT(!EVTAG_GET(reply, error_reason, &error_reason));
fprintf(stderr, "%s: channel: %s -> %d - %s\n",
dht_node_id_ascii(dht_myid(group->dht)),
channel_name, error_code, error_reason);
timerclear(&tv);
tv.tv_usec = 500000L;
event_once(-1, EV_TIMEOUT, part_cb, arg, &tv);
}
void grok_program_add_input_process(grok_program_t *gprog,
grok_input_t *ginput) {
struct bufferevent *bev;
grok_input_process_t *gipt = &(ginput->source.process);
int childin[2], childout[2], childerr[2];
int pid;
struct timeval now = { 0, 0 };
safe_pipe(childin);
safe_pipe(childout);
safe_pipe(childerr);
gipt->p_stdin = childin[1];
gipt->p_stdout = childout[0];
gipt->p_stderr = childerr[0];
gipt->c_stdin = childin[0];
gipt->c_stdout = childout[1];
gipt->c_stderr = childerr[1];
bev = bufferevent_new(gipt->p_stdout, _program_process_stdout_read,
NULL, _program_process_buferror, ginput);
bufferevent_enable(bev, EV_READ);
ginput->bev = bev;
if (gipt->read_stderr) {
/* store this somewhere */
bev = bufferevent_new(gipt->p_stderr, _program_process_stdout_read,
NULL, _program_process_buferror, ginput);
bufferevent_enable(bev, EV_READ);
}
grok_log(ginput, LOG_PROGRAMINPUT, "Scheduling start of: %s", gipt->cmd);
event_once(-1, EV_TIMEOUT, _program_process_start, ginput, &now);
}
static void web_pipeline_stream_trickle(int fd, short events, void *arg)
{
struct evbuffer *evb = NULL;
struct chunk_req_state *state = static_cast<chunk_req_state*>(arg);
struct timeval when = { 0, 0 };
long unsigned int outlen;
unsigned char *outbuf;
when.tv_usec = state->delay * 1000;
if ( state->closed ) {
// free !
state->stream->setInput(NULL);
delete state->stream;
free(state);
return;
}
if ( !state->stream->copy() ) {
event_once(-1, EV_TIMEOUT, web_pipeline_stream_trickle, state, &when);
return;
}
if ( state->stream->output_buffer->nChannels == 3 )
cvCvtColor(state->stream->output_buffer, state->stream->output_buffer, CV_BGR2RGB);
ipl2jpeg(state->stream->output_buffer, &outbuf, &outlen);
evb = evbuffer_new();
evbuffer_add_printf(evb, "--mjpegstream\r\n");
evbuffer_add_printf(evb, "Content-Type: image/jpeg\r\n");
evbuffer_add_printf(evb, "Content-Length: %lu\r\n\r\n", outlen);
evbuffer_add(evb, outbuf, outlen);
evhttp_send_reply_chunk(state->req, evb);
evbuffer_free(evb);
free(outbuf);
event_once(-1, EV_TIMEOUT, web_pipeline_stream_trickle, state, &when);
/**
evhttp_send_reply_end(state->req);
free(state);
**/
}
void _program_file_repair_event(int fd, short what, void *data) {
grok_input_t *ginput = (grok_input_t *)data;
grok_input_file_t *gift = &(ginput->source.file);
struct bufferevent *bev = ginput->bev;
struct stat st;
if (stat(gift->filename, &st) != 0) {
grok_log(ginput, LOG_PROGRAM, "Failure stat(2)'ing file '%s': %s",
gift->filename, strerror(errno));
grok_log(ginput, LOG_PROGRAM,
"Unrecoverable error (stat failed). Can't continue watching '%s'",
gift->filename);
return;
}
if (gift->st.st_ino != st.st_ino) {
/* inode changed, reopen file */
grok_log(ginput, LOG_PROGRAMINPUT,
"File inode changed from %d to %d. Reopening file '%s'",
gift->st.st_ino, st.st_ino, gift->filename);
close(gift->fd);
gift->fd = open(gift->filename, O_RDONLY);
gift->waittime.tv_sec = 0;
gift->waittime.tv_usec = 0;
gift->offset = 0;
} else if (st.st_size < gift->st.st_size) {
/* File size shrank */
grok_log(ginput, LOG_PROGRAMINPUT,
"File size shrank from %d to %d. Seeking to beginning of file '%s'",
gift->st.st_size, st.st_size, gift->filename);
gift->offset = 0;
lseek(gift->fd, gift->offset, SEEK_SET);
gift->waittime.tv_sec = 0;
gift->waittime.tv_usec = 0;
} else {
/* Nothing changed, we should wait */
if (gift->waittime.tv_sec == 0) {
gift->waittime.tv_sec = 1;
} else {
gift->waittime.tv_sec *= 2;
if (gift->waittime.tv_sec > 60) {
gift->waittime.tv_sec = 60;
}
}
}
memcpy(&(gift->st), &st, sizeof(st));
grok_log(ginput, LOG_PROGRAMINPUT,
"Repairing event with fd %d file '%s'. Will read again in %d.%d secs",
bev->ev_read.ev_fd, gift->filename,
gift->waittime.tv_sec, gift->waittime.tv_usec);
//event_add(&bev->ev_read, &(gift->waittime));
event_once(0, EV_TIMEOUT, _program_file_read_real, ginput, &(gift->waittime));
}
void fileread_real(int unused, short what, void *data) {
struct evfdpipe *ep = data;
struct timeval t = { 1, 0 };
char buf[4096];
int bytes;
bytes = read(ep->input, buf, 4096);
printf("%d => %d == %d bytes\n", ep->input, ep->output, bytes);
write(ep->output, buf, bytes);
event_once(-1, EV_TIMEOUT, fileread_real, ep, &t);
}
static void
http_large_delay_cb(struct evhttp_request *req, void *arg)
{
struct timeval tv;
timerclear(&tv);
tv.tv_sec = 3;
event_once(-1, EV_TIMEOUT, http_delay_reply, req, &tv);
/* here we close the client connection which will cause an EOF */
evhttp_connection_fail(delayed_client, EVCON_HTTP_EOF);
}
void web_pipeline_stream(struct evhttp_request *req, void *arg) {
struct timeval when = { 0, 20 };
struct evkeyvalq headers;
const char *uri;
int idx = 0;
moModule *module = NULL;
uri = evhttp_request_uri(req);
evhttp_parse_query(uri, &headers);
if ( evhttp_find_header(&headers, "objectname") == NULL ) {
evhttp_clear_headers(&headers);
return web_error(req, "missing objectname");
}
module = module_search(evhttp_find_header(&headers, "objectname"), pipeline);
if ( module == NULL ) {
evhttp_clear_headers(&headers);
return web_error(req, "object not found");
}
if ( evhttp_find_header(&headers, "index") != NULL )
idx = atoi(evhttp_find_header(&headers, "index"));
if ( idx < 0 || idx >= module->getOutputCount() ) {
evhttp_clear_headers(&headers);
return web_error(req, "invalid index");
}
struct chunk_req_state *state = (struct chunk_req_state*)malloc(sizeof(struct chunk_req_state));
memset(state, 0, sizeof(struct chunk_req_state));
state->req = req;
state->closed = false;
state->stream = new otStreamModule();
state->delay = 100;
if ( evhttp_find_header(&headers, "scale") != NULL )
state->stream->property("scale").set(evhttp_find_header(&headers, "scale"));
if ( evhttp_find_header(&headers, "delay") != NULL )
state->delay = atoi(evhttp_find_header(&headers, "delay"));
state->stream->setInput(module->getOutput(idx));
evhttp_add_header(req->output_headers, "Content-Type", "multipart/x-mixed-replace; boundary=mjpegstream");
evhttp_send_reply_start(req, HTTP_OK, "Everything is fine");
evhttp_connection_set_closecb(req->evcon, web_pipeline_stream_close, state);
event_once(-1, EV_TIMEOUT, web_pipeline_stream_trickle, state, &when);
}
int
job_thread_create (CcnetJob *job)
{
if (ccnet_pipe (job->pipefd) < 0) {
/* g_warning ("pipe error: %s\n", strerror(errno)); */
return -1;
}
g_thread_pool_push (job->manager->thread_pool, job, NULL);
#ifndef UNIT_TEST
event_once (job->pipefd[0], EV_READ, job_done_cb, job, NULL);
#endif
return 0;
}
static void
http_chunked_cb(struct evhttp_request *req, void *arg)
{
struct timeval when = { 0, 0 };
struct chunk_req_state *state = malloc(sizeof(struct chunk_req_state));
event_debug(("%s: called\n", __func__));
memset(state, 0, sizeof(struct chunk_req_state));
state->req = req;
/* generate a chunked reply */
evhttp_send_reply_start(req, HTTP_OK, "Everything is fine");
/* but trickle it across several iterations to ensure we're not
* assuming it comes all at once */
event_once(-1, EV_TIMEOUT, http_chunked_trickle_cb, state, &when);
}
void grok_program_add_input_file(grok_program_t *gprog,
grok_input_t *ginput) {
struct bufferevent *bev;
struct stat st;
int ret;
int pipefd[2];
grok_input_file_t *gift = &(ginput->source.file);
grok_log(ginput, LOG_PROGRAMINPUT, "Adding file input: %s", gift->filename);
ret = stat(gift->filename, &st);
if (ret == -1) {
grok_log(gprog, LOG_PROGRAMINPUT , "Failure stat(2)'ing file: %s",
gift->filename);
grok_log(gprog, LOG_PROGRAMINPUT , "strerror(%d): %s", strerror(errno));
return;
}
gift->fd = open(gift->filename, O_RDONLY);
if (gift->fd < 0) {
grok_log(gprog, LOG_PROGRAM, "Failure open(2)'ing file for read '%s': %s",
gift->filename, strerror(errno));
return;
}
safe_pipe(pipefd);
gift->offset = 0;
gift->reader = pipefd[0];
gift->writer = pipefd[1];
memcpy(&(gift->st), &st, sizeof(st));
gift->waittime.tv_sec = 0;
gift->waittime.tv_usec = 0;
gift->readbuffer = malloc(st.st_blksize);
grok_log(ginput, LOG_PROGRAMINPUT, "dup2(%d, %d)", gift->fd, gift->writer);
/* Tie our open file read fd to the writer of our pipe */
// this doesn't work
bev = bufferevent_new(gift->reader, _program_file_read_buffer,
NULL, _program_file_buferror, ginput);
bufferevent_enable(bev, EV_READ);
ginput->bev = bev;
event_once(-1, EV_TIMEOUT, _program_file_read_real, ginput,
&(gift->waittime));
}
static void
close_detect_cb(struct evhttp_request *req, void *arg)
{
struct evhttp_connection *evcon = arg;
struct timeval tv;
if (req != NULL && req->response_code != HTTP_OK) {
fprintf(stderr, "FAILED\n");
exit(1);
}
timerclear(&tv);
tv.tv_sec = 3; /* longer than the http time out */
/* launch a new request on the persistent connection in 6 seconds */
event_once(-1, EV_TIMEOUT, close_detect_launch, evcon, &tv);
}
void filewatch(void) {
int fd;
int p[2];
struct bufferevent *bev;
struct timeval t = { 1, 0 };
pipe(p);
fd = open("/var/log/messages", O_RDONLY);
//dup2(fd, p[1]);
bev = bufferevent_new(p[0], bufread, NULL, NULL, "fileread");
bufferevent_enable(bev, EV_READ);
struct evfdpipe *ep = calloc(1, sizeof(struct evfdpipe));
ep->input = fd;
ep->output = p[1];
event_once(p[1], EV_TIMEOUT, fileread_real, ep, &t);
}
static void
http_chunked_trickle_cb(int fd, short events, void *arg)
{
struct evbuffer *evb = evbuffer_new();
struct chunk_req_state *state = arg;
struct timeval when = { 0, 0 };
evbuffer_add_printf(evb, "%s", CHUNKS[state->i]);
evhttp_send_reply_chunk(state->req, evb);
evbuffer_free(evb);
if (++state->i < sizeof(CHUNKS)/sizeof(CHUNKS[0])) {
event_once(-1, EV_TIMEOUT,
http_chunked_trickle_cb, state, &when);
} else {
evhttp_send_reply_end(state->req);
free(state);
}
}
void _program_file_read_real(int fd, short what, void *data) {
grok_input_t *ginput = (grok_input_t *)data;
grok_input_file_t *gift = &(ginput->source.file);
grok_program_t *gprog = ginput->gprog;
int write_ret;
int bytes = 0;
bytes = read(gift->fd, gift->readbuffer, gift->st.st_blksize);
write_ret = write(gift->writer, gift->readbuffer, bytes);
if (write_ret == -1) {
grok_log(ginput, LOG_PROGRAMINPUT,
"fatal write() to pipe fd %d of %d bytes: %s",
gift->writer, bytes, strerror(errno));
/* XXX: Maybe just shutdown this particular process/file instead
* of exiting */
exit(1);
}
gift->offset += bytes;
/* we can potentially read past our last 'filesize' if the file
* has been updated since stat()'ing it. */
if (gift->offset > gift->st.st_size)
gift->st.st_size = gift->offset;
grok_log(ginput, LOG_PROGRAMINPUT, "%s: read %d bytes", gift->filename, bytes);
if (bytes == 0) { /* nothing to read, at EOF */
grok_input_eof_handler(0, 0, ginput);
} else if (bytes < 0) {
grok_log(ginput, LOG_PROGRAMINPUT, "Error: Bytes read < 0: %d", bytes);
grok_log(ginput, LOG_PROGRAMINPUT, "Error: strerror() says: %s",
strerror(errno));
} else {
/* We read more than 0 bytes, so we should keep reading this file
* immediately */
gift->waittime.tv_sec = 0;
gift->waittime.tv_usec = 0;
event_once(0, EV_TIMEOUT, _program_file_read_real, ginput,
&(gift->waittime));
}
}
void _program_file_buferror(struct bufferevent *bev, short what,
void *data) {
grok_input_t *ginput = (grok_input_t *)data;
grok_input_file_t *gift = &(ginput->source.file);
grok_program_t *gprog = ginput->gprog;
struct timeval nodelay = { 0, 0 };
grok_log(ginput, LOG_PROGRAMINPUT, "Buffer error %d on file %d: %s",
what, gift->fd, gift->filename);
if (what & EVBUFFER_EOF) {
/* EOF erro on a file, which means libevent forgets about it.
* let's re-add it */
grok_log(ginput, LOG_PROGRAMINPUT,
"EOF Error on file buffer for '%s'. Ignoring.", gift->filename);
ginput->restart_delay.tv_sec = gift->waittime.tv_sec;
ginput->restart_delay.tv_usec = gift->waittime.tv_usec;
event_once(0, EV_TIMEOUT, grok_input_eof_handler, ginput, &nodelay);
//} else if (what & EVBUFFER_TIMEOUT) {
///* Timeout reading from our file buffer */
//ginput->restart_delay.tv_sec = gift->waittime.tv_sec;
//ginput->restart_delay.tv_usec = gift->waittime.tv_usec;
//bufferevent_enable(ginput->bev, EV_READ);
}
}
static void
http_basic_test(void)
{
struct timeval tv;
struct bufferevent *bev;
int fd;
const char *http_request;
short port = -1;
test_ok = 0;
fprintf(stdout, "Testing Basic HTTP Server: ");
http = http_setup(&port, NULL);
/* bind to a second socket */
if (evhttp_bind_socket(http, "127.0.0.1", port + 1) == -1) {
fprintf(stdout, "FAILED (bind)\n");
exit(1);
}
fd = http_connect("127.0.0.1", port);
/* Stupid thing to send a request */
bev = bufferevent_new(fd, http_readcb, http_writecb,
http_errorcb, NULL);
/* first half of the http request */
http_request =
"GET /test HTTP/1.1\r\n"
"Host: some";
bufferevent_write(bev, http_request, strlen(http_request));
timerclear(&tv);
tv.tv_usec = 10000;
event_once(-1, EV_TIMEOUT, http_complete_write, bev, &tv);
event_dispatch();
if (test_ok != 3) {
fprintf(stdout, "FAILED\n");
exit(1);
}
/* connect to the second port */
bufferevent_free(bev);
EVUTIL_CLOSESOCKET(fd);
fd = http_connect("127.0.0.1", port + 1);
/* Stupid thing to send a request */
bev = bufferevent_new(fd, http_readcb, http_writecb,
http_errorcb, NULL);
http_request =
"GET /test HTTP/1.1\r\n"
"Host: somehost\r\n"
"Connection: close\r\n"
"\r\n";
bufferevent_write(bev, http_request, strlen(http_request));
event_dispatch();
bufferevent_free(bev);
EVUTIL_CLOSESOCKET(fd);
evhttp_free(http);
if (test_ok != 5) {
fprintf(stdout, "FAILED\n");
exit(1);
}
fprintf(stdout, "OK\n");
}
static void web_pipeline_stream_trickle(int fd, short events, void *arg) {
struct evbuffer *evb = NULL;
struct chunk_req_state *state = static_cast<chunk_req_state*>(arg);
struct timeval when = { 0, 0 };
long unsigned int outlen;
std::vector<uchar>outbuf;
std::vector<int> params;
IplImage* img;
bool convert = false;
when.tv_usec = state->delay * 1000;
// stream is closed, clean the state.
if ( state->closed ) {
state->stream->setInput(NULL);
delete state->stream;
free(state);
return;
}
// nothing to copy, schedule the next event
if ( !state->stream->copy() )
goto stream_trickle_end;
// convert the image from BRG to RGB
img = state->stream->output_buffer;
if ( img == NULL ) {
LOG(MO_ERROR, "stream: output_buffer is null !");
goto stream_trickle_end;
}
if ( img->nChannels == 3 )
cvCvtColor(img, img, CV_BGR2RGB);
// if the depth is not a 8, create a temporary image, and convert to 8 bytes depth
if ( img->depth != 8 ) {
convert = true;
img = cvCreateImage(cvSize(img->width, img->height), IPL_DEPTH_8U, img->nChannels);
if ( img == NULL ) {
LOG(MO_ERROR, "stream: unable to create a new image");
goto stream_trickle_end;
}
cvConvertScale(state->stream->output_buffer, img, 255, 0);
}
// convert the image to JPEG
params.push_back(CV_IMWRITE_JPEG_QUALITY);
params.push_back(100);
cv::imencode(".jpg", img, outbuf, params);
outlen = outbuf.size();
// release temporary image if created
if ( convert )
cvReleaseImage(&img);
// create and send the buffer
evb = evbuffer_new();
if ( evb == NULL ) {
LOG(MO_ERROR, "stream: unable to create a libevent buffer");
goto stream_trickle_end;
}
evbuffer_add_printf(evb, "--mjpegstream\r\n");
evbuffer_add_printf(evb, "Content-Type: image/jpeg\r\n");
evbuffer_add_printf(evb, "Content-Length: %lu\r\n\r\n", outlen);
evbuffer_add(evb, &outbuf[0], outlen);
evhttp_send_reply_chunk(state->req, evb);
evbuffer_free(evb);
outbuf.clear();
params.clear();
stream_trickle_end:;
event_once(-1, EV_TIMEOUT, web_pipeline_stream_trickle, state, &when);
}