void grok_input_eof_handler(int fd, short what, void *data) {
grok_input_t *ginput = (grok_input_t *)data;
grok_program_t *gprog = ginput->gprog;
if (ginput->instance_match_count == 0) {
/* execute nomatch if there is one on this program */
grok_matchconfig_exec_nomatch(gprog, ginput);
}
switch (ginput->type) {
case I_PROCESS:
if (PROCESS_SHOULD_RESTART(&(ginput->source.process))) {
ginput->instance_match_count = 0;
event_once(-1, EV_TIMEOUT, _program_process_start, ginput,
&ginput->restart_delay);
} else {
grok_log(ginput->gprog, LOG_PROGRAM, "Not restarting process: %s",
ginput->source.process.cmd);
bufferevent_disable(ginput->bev, EV_READ);
close(ginput->source.process.p_stdin);
close(ginput->source.process.p_stdout);
close(ginput->source.process.p_stderr);
ginput->done = 1;
}
break;
case I_FILE:
if (ginput->source.file.follow) {
ginput->instance_match_count = 0;
event_once(-1, EV_TIMEOUT, _program_file_repair_event, ginput,
&ginput->restart_delay);
} else {
grok_log(ginput->gprog, LOG_PROGRAM, "Not restarting file: %s",
ginput->source.file.filename);
bufferevent_disable(ginput->bev, EV_READ);
close(ginput->source.file.reader);
close(ginput->source.file.writer);
close(ginput->source.file.fd);
ginput->done = 1;
}
break;
}
/* If all inputs are now done, close the shell */
int still_open = 0;
int i = 0;
for (i = 0; i < gprog->ninputs; i++) {
still_open += !gprog->inputs[i].done;
if (!gprog->inputs[i].done) {
grok_log(gprog, LOG_PROGRAM, "Input still open: %d", i);
}
}
if (still_open == 0) {
for (i = 0; i < gprog->nmatchconfigs; i++) {
grok_matchconfig_close(gprog, &gprog->matchconfigs[i]);
}
grok_collection_check_end_state(gprog->gcol);
}
}
void grok_matchconfig_react(grok_program_t *gprog, grok_input_t *ginput,
grok_matchconf_t *gmc, grok_match_t *gm) {
char *reaction;
ginput->instance_match_count++;
if (gmc->no_reaction) {
grok_log(gprog, LOG_REACTION, "Reaction set to none, skipping reaction.");
return;
}
reaction = grok_matchconfig_filter_reaction(gmc->reaction, gm);
if (reaction == NULL) {
reaction = gmc->reaction;
}
if (gmc->shellinput == NULL) {
grok_matchconfig_start_shell(gprog, gmc);
}
grok_log(gprog, LOG_REACTION, "Sending '%s' to subshell", reaction);
fprintf(gmc->shellinput, "%s\n", reaction);
if (gmc->flush) {
grok_log(gprog, LOG_REACTION, "flush enabled, calling fflush");
fflush(gmc->shellinput);
}
/* This clause will occur if grok_matchconfig_filter_reaction had to do
* any meaningful work replacing %{FOO} and such */
if (reaction != gmc->reaction) {
free(reaction);
}
}
void _program_process_start(int fd, short what, void *data) {
grok_input_t *ginput = (grok_input_t*)data;
grok_input_process_t *gipt = &(ginput->source.process);
grok_program_t *gprog = ginput->gprog;
int pid = 0;
/* reset the 'instance match count' since we're starting the process */
ginput->instance_match_count = 0;
/* start the process */
pid = fork();
if (pid != 0) {
gipt->pid = pid;
gipt->pgid = getpgid(pid);
gettimeofday(&(gipt->start_time), NULL);
grok_log(ginput, LOG_PROGRAMINPUT,
"Starting process: '%s' (%d)", gipt->cmd, getpid());
return;
}
dup2(gipt->c_stdin, 0);
dup2(gipt->c_stdout, 1);
if (gipt->read_stderr) {
dup2(gipt->c_stderr, 2);
}
execlp("sh", "sh", "-c", gipt->cmd, NULL);
grok_log(ginput, LOG_PROGRAM,
"execlp(2) returned unexpectedly. Is 'sh' in your path?");
grok_log(ginput, LOG_PROGRAM, "execlp: %s", strerror(errno));
exit(-1); /* in case execlp fails */
}
void _program_file_repair_event(int fd, short what, void *data) {
grok_input_t *ginput = (grok_input_t *)data;
grok_input_file_t *gift = &(ginput->source.file);
struct bufferevent *bev = ginput->bev;
struct stat st;
if (stat(gift->filename, &st) != 0) {
grok_log(ginput, LOG_PROGRAM, "Failure stat(2)'ing file '%s': %s",
gift->filename, strerror(errno));
grok_log(ginput, LOG_PROGRAM,
"Unrecoverable error (stat failed). Can't continue watching '%s'",
gift->filename);
return;
}
if (gift->st.st_ino != st.st_ino) {
/* inode changed, reopen file */
grok_log(ginput, LOG_PROGRAMINPUT,
"File inode changed from %d to %d. Reopening file '%s'",
gift->st.st_ino, st.st_ino, gift->filename);
close(gift->fd);
gift->fd = open(gift->filename, O_RDONLY);
gift->waittime.tv_sec = 0;
gift->waittime.tv_usec = 0;
gift->offset = 0;
} else if (st.st_size < gift->st.st_size) {
/* File size shrank */
grok_log(ginput, LOG_PROGRAMINPUT,
"File size shrank from %d to %d. Seeking to beginning of file '%s'",
gift->st.st_size, st.st_size, gift->filename);
gift->offset = 0;
lseek(gift->fd, gift->offset, SEEK_SET);
gift->waittime.tv_sec = 0;
gift->waittime.tv_usec = 0;
} else {
/* Nothing changed, we should wait */
if (gift->waittime.tv_sec == 0) {
gift->waittime.tv_sec = 1;
} else {
gift->waittime.tv_sec *= 2;
if (gift->waittime.tv_sec > 60) {
gift->waittime.tv_sec = 60;
}
}
}
memcpy(&(gift->st), &st, sizeof(st));
grok_log(ginput, LOG_PROGRAMINPUT,
"Repairing event with fd %d file '%s'. Will read again in %d.%d secs",
bev->ev_read.ev_fd, gift->filename,
gift->waittime.tv_sec, gift->waittime.tv_usec);
//event_add(&bev->ev_read, &(gift->waittime));
event_once(0, EV_TIMEOUT, _program_file_read_real, ginput, &(gift->waittime));
}
static void grok_capture_add_predicate(grok_t *grok, int capture_id,
const char *predicate, int predicate_len) {
grok_capture *gct;
int offset = 0;
grok_log(grok, LOG_PREDICATE, "Adding predicate '%.*s' to capture %d",
predicate_len, predicate, capture_id);
gct = (grok_capture *)grok_capture_get_by_id(grok, capture_id);
if (gct == NULL) {
grok_log(grok, LOG_PREDICATE, "Failure to find capture id %d", capture_id);
return;
}
/* Compile the predicate into something useful */
/* So far, predicates are just an operation and an argument */
/* TODO(sissel): add predicate_func(capture_str, args) ??? */
/* skip leading whitespace, use a loop since 'strspn' doesn't take a len */
while (isspace(predicate[offset]) && offset < predicate_len) {
offset++;
}
predicate += offset;
predicate_len -= offset;
if (predicate_len > 2) {
if (!strncmp(predicate, "=~", 2) || !strncmp(predicate, "!~", 2)) {
grok_predicate_regexp_init(grok, gct, predicate, predicate_len);
return;
} else if ((predicate[0] == '$')
&& (strchr("!<>=", predicate[1]) != NULL)) {
grok_predicate_strcompare_init(grok, gct, predicate, predicate_len);
return;
}
}
if (predicate_len > 1) {
if (strchr("!<>=", predicate[0]) != NULL) {
grok_predicate_numcompare_init(grok, gct, predicate, predicate_len);
} else {
fprintf(stderr, "Invalid predicate: '%.*s'\n", predicate_len, predicate);
}
} else {
/* predicate_len == 1, here, and no 1-character predicates exist */
fprintf(stderr, "Invalid predicate: '%.*s'\n", predicate_len, predicate);
}
/* update the database with our modified grok_capture */
grok_capture_add(grok, gct);
}
static void grok_study_capture_map(grok_t *grok) {
char *nametable;
grok_capture *gct;
int nametable_size;
int nametable_entrysize;
int i = 0;
int offset = 0;
int stringnum;
int capture_id;
pcre_fullinfo(grok->re, NULL, PCRE_INFO_NAMECOUNT, &nametable_size);
pcre_fullinfo(grok->re, NULL, PCRE_INFO_NAMEENTRYSIZE, &nametable_entrysize);
pcre_fullinfo(grok->re, NULL, PCRE_INFO_NAMETABLE, &nametable);
for (i = 0; i < nametable_size; i++) {
offset = i * nametable_entrysize;
stringnum = (nametable[offset] << 8) + nametable[offset + 1];
sscanf(nametable + offset + 2, CAPTURE_FORMAT, &capture_id);
grok_log(grok, LOG_COMPILE, "Studying capture %d", capture_id);
gct = (grok_capture *)grok_capture_get_by_id(grok, capture_id);
assert(gct != NULL);
gct->pcre_capture_number = stringnum;
/* update the database with the new data */
grok_capture_add(grok, gct);
}
}
int grok_patterns_import_from_string(const grok_t *grok, const char *buffer) {
char *tokctx = NULL;
char *tok = NULL;
char *strptr = NULL;
char *dupbuf = NULL;
grok_log(grok, LOG_PATTERNS, "Importing patterns from string");
dupbuf = strdup(buffer);
strptr = dupbuf;
while ((tok = strtok_r(strptr, "\n", &tokctx)) != NULL) {
const char *name, *regexp;
size_t name_len, regexp_len;
strptr = NULL;
/* skip leading whitespace */
tok += strspn(tok, " \t");
/* If first non-whitespace is a '#', then this is a comment. */
if (*tok == '#') continue;
_pattern_parse_string(tok, &name, &name_len, ®exp, ®exp_len);
(void) grok_pattern_add(grok, name, name_len, regexp, regexp_len);
}
free(dupbuf);
return GROK_OK;
}
int grok_pattern_find(const grok_t *grok, const char *name, size_t name_len,
const char **regexp, size_t *regexp_len) {
TCTREE *patterns = grok->patterns;
*regexp = tctreeget(patterns, name, name_len, (int*) regexp_len);
grok_log(grok, LOG_PATTERNS, "Searching for pattern '%s' (%s): %.*s",
name, *regexp == NULL ? "not found" : "found", *regexp_len, *regexp);
if (*regexp == NULL) {
grok_log(grok, LOG_PATTERNS, "pattern '%s': not found", name);
*regexp = NULL;
*regexp_len = 0;
return GROK_ERROR_PATTERN_NOT_FOUND;
}
return GROK_OK;
}
void grok_program_add_input_process(grok_program_t *gprog,
grok_input_t *ginput) {
struct bufferevent *bev;
grok_input_process_t *gipt = &(ginput->source.process);
int childin[2], childout[2], childerr[2];
int pid;
struct timeval now = { 0, 0 };
safe_pipe(childin);
safe_pipe(childout);
safe_pipe(childerr);
gipt->p_stdin = childin[1];
gipt->p_stdout = childout[0];
gipt->p_stderr = childerr[0];
gipt->c_stdin = childin[0];
gipt->c_stdout = childout[1];
gipt->c_stderr = childerr[1];
bev = bufferevent_new(gipt->p_stdout, _program_process_stdout_read,
NULL, _program_process_buferror, ginput);
bufferevent_enable(bev, EV_READ);
ginput->bev = bev;
if (gipt->read_stderr) {
/* store this somewhere */
bev = bufferevent_new(gipt->p_stderr, _program_process_stdout_read,
NULL, _program_process_buferror, ginput);
bufferevent_enable(bev, EV_READ);
}
grok_log(ginput, LOG_PROGRAMINPUT, "Scheduling start of: %s", gipt->cmd);
event_once(-1, EV_TIMEOUT, _program_process_start, ginput, &now);
}
void grok_discover_init(grok_discover_t *gdt, grok_t *source_grok) {
TCLIST *names = NULL;
int i = 0, len = 0;
if (dgrok_init == 0) {
grok_discover_global_init();
}
gdt->complexity_tree = tctreenew2(tccmpint32, NULL);
gdt->base_grok = source_grok;
gdt->logmask = source_grok->logmask;
gdt->logdepth = source_grok->logdepth;
names = grok_pattern_name_list(source_grok);
len = tclistnum(names);
/* for each pattern, create a grok.
* Sort by complexity.
* loop
* for each pattern, try replacement
* if no replacements, break
*/
for (i = 0; i < len; i++) {
int namelen = 0;
const char *name = tclistval(names, i, &namelen);
int *key = malloc(sizeof(int));
grok_t *g = grok_new();
grok_clone(g, source_grok);
char *gpattern;
//if (asprintf(&gpattern, "%%{%.*s =~ /\\b/}", namelen, name) == -1) {
if (asprintf(&gpattern, "%%{%.*s}", namelen, name) == -1) {
perror("asprintf failed");
abort();
}
grok_compile(g, gpattern);
*key = complexity(g);
/* Low complexity should be skipped */
if (*key > -20) {
free((void *)g->pattern);
free(key);
grok_free_clone(g);
free(g);
continue;
}
*key *= 1000; /* Inflate so we can insert duplicates */
grok_log(gdt, LOG_DISCOVER, "Including pattern: (complexity: %d) %.*s",
*(int *)key, namelen, name);
while (!tctreeputkeep(gdt->complexity_tree, key, sizeof(int),
g, sizeof(grok_t))) {
*key--;
}
//grok_free_clone(g);
//free(key);
}
tclistdel(names);
}
const grok_capture *grok_capture_walk_next(const TCTREE_ITER *iter, const grok_t *grok) {
int id_size;
int gct_size;
int *id;
const grok_capture *gct;
id = (int *)tctreeiternext(iter, &id_size);
if (id == NULL) {
grok_log(grok, LOG_CAPTURE, "walknext null");
return NULL;
}
grok_log(grok, LOG_CAPTURE, "walknext ok %d", *id);
gct = (grok_capture *)tctreeget(grok->captures_by_id, id, id_size,
&gct_size);
return gct;
}
void _program_process_buferror(struct bufferevent *bev, short what,
void *data) {
grok_input_t *ginput = (grok_input_t *)data;
grok_input_process_t *gipt = &(ginput->source.process);
grok_program_t *gprog = ginput->gprog;
grok_log(ginput, LOG_PROGRAMINPUT, "Buffer error %d on process %d: %s",
what, gipt->pid, gipt->cmd);
}
static int grok_pcre_callout(pcre_callout_block *pcb) {
grok_t *grok = pcb->callout_data;
const grok_capture *gct;
gct = (grok_capture *)grok_capture_get_by_capture_number(grok,
pcb->capture_last);
/* TODO(sissel): handle case where gct is not found (== NULL) */
if (gct->predicate_func_name != NULL) {
int (*predicate)(grok_t *, const grok_capture *, const char *, int, int);
int start, end;
void *handle;
char *lib = gct->predicate_lib;
start = pcb->offset_vector[ pcb->capture_last * 2 ];
end = pcb->offset_vector[ pcb->capture_last * 2 + 1];
if (lib != NULL && lib[0] == '\0') {
lib = NULL;
}
/* Hack so if we are dlopen()'d we can still find ourselves
* This is necessary for cases like Ruby FFI which only dlopen's us
* and does not explicitly link/load us? */
lib = "libgrok.so";
handle = dlopen(lib, RTLD_LAZY);
predicate = dlsym(handle, gct->predicate_func_name);
if (predicate != NULL) {
grok_log(grok, LOG_EXEC, "start pcre_callout func %s/%.*s",
(lib == NULL ? "grok" : lib), gct->predicate_func_name_len,
gct->predicate_func_name);
int ret;
ret = predicate(grok, gct, pcb->subject, start, end);
grok_log(grok, LOG_EXEC, "end pcre_callout func %s/%.*s returned: %d",
(lib == NULL ? "grok" : lib), gct->predicate_func_name_len,
gct->predicate_func_name, ret);
return ret;
} else {
grok_log(grok, LOG_EXEC, "No such function '%s' in library '%s'",
gct->predicate_func_name, lib);
return 0;
}
}
return 0;
} /* int grok_pcre_callout */
int grok_execn(const grok_t *grok, const char *text, int textlen, grok_match_t *gm) {
int ret;
pcre_extra pce;
pce.flags = PCRE_EXTRA_CALLOUT_DATA;
pce.callout_data = (void *)grok;
if (grok->re == NULL) {
grok_log(grok, LOG_EXEC, "Error: pcre re is null, meaning you haven't called grok_compile yet");
fprintf(stderr, "ERROR: grok_execn called on an object that has not pattern compiled. Did you call grok_compile yet?\n");
return GROK_ERROR_UNINITIALIZED;
}
ret = pcre_exec(grok->re, &pce, text, textlen, 0, 0,
grok->pcre_capture_vector, grok->pcre_num_captures * 3);
grok_log(grok, LOG_EXEC, "%.*s =~ /%s/ => %d",
textlen, text, grok->pattern, ret);
if (ret < 0) {
switch (ret) {
case PCRE_ERROR_NOMATCH:
return GROK_ERROR_NOMATCH;
break;
case PCRE_ERROR_NULL:
fprintf(stderr, "Null error, one of the arguments was null?\n");
break;
case PCRE_ERROR_BADOPTION:
fprintf(stderr, "pcre badoption\n");
break;
case PCRE_ERROR_BADMAGIC:
fprintf(stderr, "pcre badmagic\n");
break;
}
//grok->pcre_errno = ret;
return GROK_ERROR_PCRE_ERROR;
}
/* Push match info into gm only if it is non-NULL */
if (gm != NULL) {
gm->grok = grok;
gm->subject = text;
gm->start = grok->pcre_capture_vector[0];
gm->end = grok->pcre_capture_vector[1];
}
return GROK_OK;
}
void grok_program_add_input_file(grok_program_t *gprog,
grok_input_t *ginput) {
struct bufferevent *bev;
struct stat st;
int ret;
int pipefd[2];
grok_input_file_t *gift = &(ginput->source.file);
grok_log(ginput, LOG_PROGRAMINPUT, "Adding file input: %s", gift->filename);
ret = stat(gift->filename, &st);
if (ret == -1) {
grok_log(gprog, LOG_PROGRAMINPUT , "Failure stat(2)'ing file: %s",
gift->filename);
grok_log(gprog, LOG_PROGRAMINPUT , "strerror(%d): %s", strerror(errno));
return;
}
gift->fd = open(gift->filename, O_RDONLY);
if (gift->fd < 0) {
grok_log(gprog, LOG_PROGRAM, "Failure open(2)'ing file for read '%s': %s",
gift->filename, strerror(errno));
return;
}
safe_pipe(pipefd);
gift->offset = 0;
gift->reader = pipefd[0];
gift->writer = pipefd[1];
memcpy(&(gift->st), &st, sizeof(st));
gift->waittime.tv_sec = 0;
gift->waittime.tv_usec = 0;
gift->readbuffer = malloc(st.st_blksize);
grok_log(ginput, LOG_PROGRAMINPUT, "dup2(%d, %d)", gift->fd, gift->writer);
/* Tie our open file read fd to the writer of our pipe */
// this doesn't work
bev = bufferevent_new(gift->reader, _program_file_read_buffer,
NULL, _program_file_buferror, ginput);
bufferevent_enable(bev, EV_READ);
ginput->bev = bev;
event_once(-1, EV_TIMEOUT, _program_file_read_real, ginput,
&(gift->waittime));
}
int grok_pattern_add(const grok_t *grok, const char *name, size_t name_len,
const char *regexp, size_t regexp_len) {
TCTREE *patterns = grok->patterns;
grok_log(grok, LOG_PATTERNS, "Adding new pattern '%.*s' => '%.*s'",
name_len, name, regexp_len, regexp);
tctreeput(patterns, name, name_len, regexp, regexp_len);
return GROK_OK;
}
void _program_file_read_real(int fd, short what, void *data) {
grok_input_t *ginput = (grok_input_t *)data;
grok_input_file_t *gift = &(ginput->source.file);
grok_program_t *gprog = ginput->gprog;
int write_ret;
int bytes = 0;
bytes = read(gift->fd, gift->readbuffer, gift->st.st_blksize);
write_ret = write(gift->writer, gift->readbuffer, bytes);
if (write_ret == -1) {
grok_log(ginput, LOG_PROGRAMINPUT,
"fatal write() to pipe fd %d of %d bytes: %s",
gift->writer, bytes, strerror(errno));
/* XXX: Maybe just shutdown this particular process/file instead
* of exiting */
exit(1);
}
gift->offset += bytes;
/* we can potentially read past our last 'filesize' if the file
* has been updated since stat()'ing it. */
if (gift->offset > gift->st.st_size)
gift->st.st_size = gift->offset;
grok_log(ginput, LOG_PROGRAMINPUT, "%s: read %d bytes", gift->filename, bytes);
if (bytes == 0) { /* nothing to read, at EOF */
grok_input_eof_handler(0, 0, ginput);
} else if (bytes < 0) {
grok_log(ginput, LOG_PROGRAMINPUT, "Error: Bytes read < 0: %d", bytes);
grok_log(ginput, LOG_PROGRAMINPUT, "Error: strerror() says: %s",
strerror(errno));
} else {
/* We read more than 0 bytes, so we should keep reading this file
* immediately */
gift->waittime.tv_sec = 0;
gift->waittime.tv_usec = 0;
event_once(0, EV_TIMEOUT, _program_file_read_real, ginput,
&(gift->waittime));
}
}
void grok_matchconfig_exec_nomatch(grok_program_t *gprog, grok_input_t *ginput) {
int i = 0;
for (i = 0; i < gprog->nmatchconfigs; i++) {
grok_matchconf_t *gmc = &gprog->matchconfigs[i];
if (gmc->is_nomatch) {
grok_log(gprog, LOG_PROGRAM, "Executing reaction for nomatch: %s",
gmc->reaction);
grok_matchconfig_react(gprog, ginput, gmc, NULL);
}
}
}
int grok_patterns_import_from_file(const grok_t *grok, const char *filename) {
FILE *patfile = NULL;
size_t filesize = 0;
size_t bytes = 0;
char *buffer = NULL;
grok_log(grok, LOG_PATTERNS, "Importing pattern file: '%s'", filename);
patfile = fopen(filename, "r");
if (patfile == NULL) {
grok_log(grok, LOG_PATTERNS, "Unable to open '%s' for reading: %s",
filename, strerror(errno));
return GROK_ERROR_FILE_NOT_ACCESSIBLE;
}
fseek(patfile, 0, SEEK_END);
filesize = ftell(patfile);
fseek(patfile, 0, SEEK_SET);
buffer = calloc(1, filesize + 1);
if (buffer == NULL) {
fprintf(stderr, "Fatal: calloc(1, %zd) failed while trying to read '%s'",
filesize, filename);
abort();
}
memset(buffer, 0, filesize);
bytes = fread(buffer, 1, filesize, patfile);
if (bytes != filesize) {
grok_log(grok, LOG_PATTERNS, "Unable to open '%s' for reading: %s",
filename, strerror(errno));
fprintf(stderr, "Expected %zd bytes, but read %zd.", filesize, bytes);
return GROK_ERROR_UNEXPECTED_READ_SIZE;
}
grok_patterns_import_from_string(grok, buffer);
free(buffer);
fclose(patfile);
return GROK_OK;
}
int grok_compilen(grok_t *grok, const char *pattern, int length) {
grok_log(grok, LOG_COMPILE, "Compiling '%.*s'", length, pattern);
/* clear the old tctree data */
tctreeclear(grok->captures_by_name);
tctreeclear(grok->captures_by_subname);
tctreeclear(grok->captures_by_capture_number);
tctreeclear(grok->captures_by_id);
grok->pattern = pattern;
grok->pattern_len = length;
grok->full_pattern = grok_pattern_expand(grok);
if (grok->full_pattern == NULL) {
grok_log(grok, LOG_COMPILE, "A failure occurred while compiling '%.*s'",
length, pattern);
grok->errstr = "failure occurred while expanding pattern "\
"(too pattern recursion?)";
return GROK_ERROR_COMPILE_FAILED;
}
grok->re = pcre_compile(grok->full_pattern, 0,
&grok->pcre_errptr, &grok->pcre_erroffset, NULL);
if (grok->re == NULL) {
grok->errstr = (char *)grok->pcre_errptr;
return GROK_ERROR_COMPILE_FAILED;
}
pcre_fullinfo(grok->re, NULL, PCRE_INFO_CAPTURECOUNT, &grok->pcre_num_captures);
grok->pcre_num_captures++; /* include the 0th group */
grok->pcre_capture_vector = calloc(3 * grok->pcre_num_captures, sizeof(int));
/* Walk grok->captures_by_id.
* For each, ask grok->re what stringnum it is */
grok_study_capture_map(grok);
return GROK_OK;
}
int grok_match_get_named_substring(const grok_match_t *gm, const char *name,
const char **substr, int *len) {
int start, end;
const grok_capture *gct;
grok_log(gm->grok, LOG_MATCH, "Fetching named capture: %s", name);
gct = grok_match_get_named_capture(gm, name);
if (gct == NULL) {
grok_log(gm->grok, LOG_MATCH, "Named capture '%s' not found", name);
*substr = NULL;
*len = 0;
return -1;
}
start = (gm->grok->pcre_capture_vector[gct->pcre_capture_number * 2]);
end = (gm->grok->pcre_capture_vector[gct->pcre_capture_number * 2 + 1]);
grok_log(gm->grok, LOG_MATCH, "Capture '%s' == '%.*s' is %d -> %d of string '%s'",
name, end - start, gm->subject + start, start, end, gm->subject);
*substr = gm->subject + start;
*len = (end - start);
return 0;
}
void grok_matchconfig_start_shell(grok_program_t *gprog,
grok_matchconf_t *gmc) {
int pipefd[2];
if (!strcmp(gmc->shell, "stdout")) {
/* Special case: Use the stdout FILE */
grok_log(gprog, LOG_PROGRAM,
"matchconfig subshell set to 'stdout', directing reaction " \
"output to stdout instead of a process.");
gmc->shellinput = stdout;
return;
}
safe_pipe(pipefd);
grok_log(gprog, LOG_PROGRAM, "Starting matchconfig subshell: %s",
(gmc->shell == NULL) ? "/bin/sh" : gmc->shell);
gmc->pid = fork();
if (gmc->pid == 0) {
close(pipefd[1]); /* close the stdin input from the parent */
/* child */
dup2(pipefd[0], 0);
if (gmc->shell == NULL) {
/* default to just sh if there's no shell set */
execlp("sh", "sh", NULL);
} else {
execlp("sh", "sh", "-c", gmc->shell, NULL);
}
fprintf(stderr, "!!! Shouldn't have gotten here. execlp failed");
perror("errno says");
exit(-1);;
}
gmc->shellinput = fdopen(pipefd[1], "w");
if (gmc->shellinput == NULL) {
grok_log(gprog, LOG_PROGRAM, "Fatal: Unable to fdopen(%d) subshell pipe: %s",
pipefd[1], strerror(errno));
exit(1); /* XXX: We shouldn't exit here, but what else should we do? */
}
}
void grok_program_add_input(grok_program_t *gprog, grok_input_t *ginput) {
grok_log(gprog, LOG_PROGRAM, "Adding input of type %s",
(ginput->type == I_FILE) ? "file" : "process");
ginput->instance_match_count = 0;
ginput->done = 0;
switch (ginput->type) {
case I_FILE:
grok_program_add_input_file(gprog, ginput);
break;
case I_PROCESS:
grok_program_add_input_process(gprog, ginput);
break;
}
}
void _program_file_buferror(struct bufferevent *bev, short what,
void *data) {
grok_input_t *ginput = (grok_input_t *)data;
grok_input_file_t *gift = &(ginput->source.file);
grok_program_t *gprog = ginput->gprog;
struct timeval nodelay = { 0, 0 };
grok_log(ginput, LOG_PROGRAMINPUT, "Buffer error %d on file %d: %s",
what, gift->fd, gift->filename);
if (what & EVBUFFER_EOF) {
/* EOF erro on a file, which means libevent forgets about it.
* let's re-add it */
grok_log(ginput, LOG_PROGRAMINPUT,
"EOF Error on file buffer for '%s'. Ignoring.", gift->filename);
ginput->restart_delay.tv_sec = gift->waittime.tv_sec;
ginput->restart_delay.tv_usec = gift->waittime.tv_usec;
event_once(0, EV_TIMEOUT, grok_input_eof_handler, ginput, &nodelay);
//} else if (what & EVBUFFER_TIMEOUT) {
///* Timeout reading from our file buffer */
//ginput->restart_delay.tv_sec = gift->waittime.tv_sec;
//ginput->restart_delay.tv_usec = gift->waittime.tv_usec;
//bufferevent_enable(ginput->bev, EV_READ);
}
}
int grok_capture_set_extra(grok_t *grok, grok_capture *gct, void *extra) {
/* Store the pointer to extra.
* XXX: This is potentially bad voodoo. */
grok_log(grok, LOG_CAPTURE, "Setting extra value of 0x%x", extra);
/* We could copy it this way, but if you compile with -fomit-frame-pointer,
* this data is lost since extra is in the stack. Copy the pointer instead.
* TODO(sissel): See if we don't need this anymore since using tokyocabinet.
*/
//gct->extra.extra_val = (char *)&extra;
gct->extra.extra_len = sizeof(void *); /* allocate pointer size */
gct->extra.extra_val = malloc(gct->extra.extra_len);
memcpy(gct->extra.extra_val, &extra, gct->extra.extra_len);
//gct->extra.extra_len = extra;
//gct->extra.extra_val = extra;
return 0;
}
void grok_matchconfig_close(grok_program_t *gprog, grok_matchconf_t *gmc) {
int ret = 0;
if (gmc->shellinput != NULL) {
/* Don't close stdout */
if (gmc->shellinput != stdout) {
ret = fclose(gmc->shellinput);
grok_log(gprog, LOG_PROGRAM, "Closing matchconf shell. fclose() = %d", ret);
}
gmc->shellinput = NULL;
}
int i = 0;
for (i = 0; i < tclistnum(gmc->grok_list); i++) {
int len;
grok_t *grok = (grok_t *) tclistval(gmc->grok_list, i, &len);
grok_free(grok);
}
tclistdel(gmc->grok_list);
}
void grok_matchconfig_exec(grok_program_t *gprog, grok_input_t *ginput,
const char *text) {
grok_match_t gm;
grok_matchconf_t *gmc;
int i = 0;
int want_break = 0;
for (i = 0; i < gprog->nmatchconfigs; i++) {
int ret;
gmc = &gprog->matchconfigs[i];
int num_groks = tclistnum(gmc->grok_list);
int i = 0;
if (gmc->is_nomatch) {
continue;
}
for (i = 0; i < num_groks; i++) {
grok_t *grok;
int unused_len;
grok = (grok_t *)tclistval(gmc->grok_list, i, &unused_len);
grok_log(gprog, LOG_PROGRAM, "Trying match against pattern %d: %.*s",
i, grok->pattern_len, grok->pattern);
ret = grok_exec(grok, text, &gm);
if (ret == GROK_OK) {
grok_matchconfig_react(gprog, ginput, gmc, &gm);
if (!gmc->no_reaction) {
gprog->reactions += 1;
}
if (gmc->break_if_match) {
want_break = 1;
break;
}
}
}
if (want_break) {
break;
}
}
}
/* WARNING - For the purposes of reading named groups into Go and taking copies of these
* strings, this is definitely safe. Re-evaluate and Valgrind if we start passing the name
* pointer instead of immediately calling CString() on it. */
int grok_match_walk_next(grok_match_t *gm,
char **name, int *namelen,
const char **substr, int *substrlen) {
const grok_capture *gct;
int start, end;
gct = grok_capture_walk_next(gm->iter, gm->grok);
if (gct == NULL) {
return 1;
}
*namelen = gct->name_len;
*name = gct->name;
start = (gm->pcre_capture_vector[gct->pcre_capture_number * 2]);
end = (gm->pcre_capture_vector[gct->pcre_capture_number * 2 + 1]);
grok_log(gm->grok, LOG_MATCH, "CaptureWalk '%.*s' is %d -> %d of string '%s'",
*namelen, *name, start, end, gm->subject);
*substr = gm->subject + start;
*substrlen = (end - start);
return 0;
}
int grok_compilen(grok_t *grok, const char *pattern, int length) {
grok_log(grok, LOG_COMPILE, "Compiling '%s'", pattern);
grok->pattern = pattern;
grok->full_pattern = grok_pattern_expand(grok); //, 0, strlen(pattern));
grok->re = pcre_compile(grok->full_pattern, 0,
&grok->pcre_errptr, &grok->pcre_erroffset,
NULL);
if (grok->re == NULL) {
grok->errstr = (char *)grok->pcre_errptr;
return GROK_ERROR_COMPILE_FAILED;
}
pcre_fullinfo(grok->re, NULL, PCRE_INFO_CAPTURECOUNT, &grok->pcre_num_captures);
grok->pcre_num_captures++; /* include the 0th group */
grok->pcre_capture_vector = calloc(3 * grok->pcre_num_captures, sizeof(int));
/* Walk grok->captures_by_id.
* For each, ask grok->re what stringnum it is */
grok_study_capture_map(grok);
return GROK_OK;
}
char *grok_match_reaction_apply_filter(grok_match_t *gm,
char **value, int *value_len,
const char *filter, int filter_len) {
int offset = 0, len = 0;
int value_size;
int ret;
struct filter *filterobj;
if (filter_len == 0) {
return *value;
}
*value = string_ndup(*value, *value_len);
/* we'll use the value_len from the function arguments */
value_size = *value_len + 1;
/* skip first char which must be a '|' */
offset = 1;
while (offset + len < filter_len) {
if (filter[offset + len] == '|') {
/* Apply the filter */
grok_log(gm->grok, LOG_REACTION, "ApplyFilter code: %.*s",
len, filter + offset);
filterobj = string_filter_lookup(filter + offset, len);
if (filterobj == NULL) {
grok_log(gm->grok, LOG_REACTION,
"Can't apply filter '%.*s'; it's unknown.",
len, filter + offset);
} else {
ret = filterobj->func(gm, value, value_len, &value_size);
if (ret != 0) {
grok_log(gm->grok, LOG_REACTION,
"Applying filter '%.*s' returned error %d for string '%.*s'.",
len, filter + offset, *value_len, *value);
}
}
offset += len + 1;
len = 0;
}
len++;
}
/* We'll always have one filter left over */
grok_log(gm->grok, LOG_REACTION, "Filter code: %.*s", len, filter + offset);
filterobj = string_filter_lookup(filter + offset, len);
if (filterobj == NULL) {
grok_log(gm->grok, LOG_REACTION,
"Can't apply filter '%.*s'; it's unknown.",
len, filter + offset);
} else {
ret = filterobj->func(gm, value, value_len, &value_size);
if (ret != 0) {
grok_log(gm->grok, LOG_REACTION,
"Applying filter '%.*s' returned error %d for string '%.*s'.",
len, filter + offset, *value_len, *value);
}
}
return *value;
}
